diff --git a/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/config.json b/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/config.json
new file mode 100644
index 0000000000000000000000000000000000000000..424c56db43fd50dd677a438fbcc7fe52824159a7
--- /dev/null
+++ b/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/config.json
@@ -0,0 +1,66 @@
+{
+ "architectures": [
+ "Qwen3NextForCausalLM"
+ ],
+ "attention_bias": true,
+ "attention_dropout": 0.0,
+ "attn_output_gate": false,
+ "auto_map": {
+ "AutoConfig": "configuration_qwen3_next.Qwen3NextConfig",
+ "AutoModel": "modeling_qwen3_next.Qwen3NextForCausalLM",
+ "AutoModelForCausalLM": "modeling_qwen3_next.Qwen3NextForCausalLM"
+ },
+ "bos_token_id": 1,
+ "decoder_sparse_step": 1,
+ "dtype": "float32",
+ "enable_qk_norm": false,
+ "eos_token_id": 2,
+ "full_attention_interval": 0,
+ "head_dim": 64,
+ "hidden_act": "silu",
+ "hidden_size": 1920,
+ "initializer_range": 0.02,
+ "intermediate_size": 4800,
+ "layer_types": ["linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "full_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "full_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention"],
+ "linear_conv_kernel_dim": 4,
+ "linear_key_head_dim": 64,
+ "linear_num_key_heads": 8,
+ "linear_num_value_heads": 32,
+ "linear_value_head_dim": 64,
+ "max_position_embeddings": 32768,
+ "mlp_only_layers": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55],
+ "num_experts_per_tok": 2,
+ "num_experts": 0,
+ "model_type": "qwen3_next",
+ "moe_intermediate_size": 0,
+ "norm_topk_prob": true,
+ "num_attention_heads": 30,
+ "num_hidden_layers": 56,
+ "num_key_value_heads": 6,
+ "output_router_logits": false,
+ "partial_rotary_factor": 1.0,
+ "rms_norm_eps": 1e-06,
+ "rope_scaling": null,
+ "rope_theta": 490000,
+ "router_aux_loss_coef": 0.001,
+ "router_bias": false,
+ "moe_router_score_function": "softmax",
+ "shared_expert_intermediate_size": 0,
+ "use_shared_expert_gate": true,
+ "tie_word_embeddings": false,
+ "transformers_version": "4.57.1",
+ "use_cache": true,
+ "use_sliding_window": false,
+ "ffn_token_shift": null,
+ "ffn_intermediate_token_shift": null,
+ "attn_token_shift": null,
+ "attn_q_token_shift": null,
+ "attn_k_token_shift": null,
+ "attn_v_token_shift": null,
+ "token_shift_conv_size": 4,
+ "token_shift_conv_init": "default",
+ "attn_position_embedding_type": "rope",
+ "rnn_position_embedding_type": "nope",
+ "attn_logits_scaling": null,
+ "vocab_size": 99000
+}
\ No newline at end of file
diff --git a/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/configuration_qwen3_next.py b/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/configuration_qwen3_next.py
new file mode 100644
index 0000000000000000000000000000000000000000..7f41fdfa8d7f284394825ebb29537c2b902a5304
--- /dev/null
+++ b/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/configuration_qwen3_next.py
@@ -0,0 +1,329 @@
+# coding=utf-8
+# Copyright 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Qwen3-Next model configuration"""
+
+from transformers.configuration_utils import PretrainedConfig, layer_type_validation
+from transformers.modeling_rope_utils import rope_config_validation
+from transformers.utils import logging
+
+
+logger = logging.get_logger(__name__)
+
+
+class Qwen3NextConfig(PretrainedConfig):
+ r"""
+ This is the configuration class to store the configuration of a [`Qwen3NextModel`]. It is used to instantiate a
+ Qwen3-Next model according to the specified arguments, defining the model architecture.
+ Instantiating a configuration with the defaults will yield a similar configuration to that of
+ Qwen3-Next-80B-A3B-Instruct [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct).
+
+ Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
+ documentation from [`PretrainedConfig`] for more information.
+
+
+ Args:
+ vocab_size (`int`, *optional*, defaults to 151936):
+ Vocabulary size of the model. Defines the number of different tokens that can be represented by the
+ `inputs_ids`.
+ hidden_size (`int`, *optional*, defaults to 2048):
+ Dimension of the hidden representations.
+ intermediate_size (`int`, *optional*, defaults to 5632):
+ Dimension of the MLP representations.
+ num_hidden_layers (`int`, *optional*, defaults to 48):
+ Number of hidden layers in the Transformer encoder.
+ num_attention_heads (`int`, *optional*, defaults to 16):
+ Number of attention heads for each attention layer in the Transformer encoder.
+ num_key_value_heads (`int`, *optional*, defaults to 2):
+ This is the number of key_value heads that should be used to implement Grouped Query Attention. If
+ `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
+ `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
+ converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
+ by meanpooling all the original heads within that group. For more details checkout [this
+ paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
+ hidden_act (`str`, *optional*, defaults to `"silu"`):
+ The non-linear activation function in the decoder.
+ max_position_embeddings (`int`, *optional*, defaults to 32768):
+ The maximum sequence length that this model might ever be used with.
+ initializer_range (`float`, *optional*, defaults to 0.02):
+ The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
+ rms_norm_eps (`float`, *optional*, defaults to 1e-06):
+ The epsilon used by the rms normalization layers.
+ use_cache (`bool`, *optional*, defaults to `True`):
+ Whether or not the model should return the last key/values attentions (not used by all models). Only
+ relevant if `config.is_decoder=True`.
+ tie_word_embeddings (`bool`, *optional*, defaults to `False`):
+ Whether the model's input and output word embeddings should be tied.
+ rope_theta (`float`, *optional*, defaults to 10000.0):
+ The base period of the RoPE embeddings.
+ rope_scaling (`Dict`, *optional*):
+ Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
+ and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
+ accordingly.
+ Expected contents:
+ `rope_type` (`str`):
+ The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
+ 'llama3'], with 'default' being the original RoPE implementation.
+ `factor` (`float`, *optional*):
+ Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
+ most scaling types, a `factor` of x will enable the model to handle sequences of length x *
+ original maximum pre-trained length.
+ `original_max_position_embeddings` (`int`, *optional*):
+ Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
+ pretraining.
+ `attention_factor` (`float`, *optional*):
+ Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
+ computation. If unspecified, it defaults to value recommended by the implementation, using the
+ `factor` field to infer the suggested value.
+ `beta_fast` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
+ ramp function. If unspecified, it defaults to 32.
+ `beta_slow` (`float`, *optional*):
+ Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
+ ramp function. If unspecified, it defaults to 1.
+ `short_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to short contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `long_factor` (`List[float]`, *optional*):
+ Only used with 'longrope'. The scaling factor to be applied to long contexts (<
+ `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
+ size divided by the number of attention heads divided by 2
+ `low_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
+ `high_freq_factor` (`float`, *optional*):
+ Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
+ partial_rotary_factor (`float`, *optional*, defaults to 0.25):
+ Percentage of the query and keys which will have rotary embedding.
+ attention_bias (`bool`, *optional*, defaults to `False`):
+ Whether to use a bias in the query, key, value and output projection layers during self-attention.
+ attention_dropout (`float`, *optional*, defaults to 0.0):
+ The dropout ratio for the attention probabilities.
+ head_dim (`int`, *optional*, defaults to 256):
+ Projection weights dimension in multi-head attention.
+ linear_conv_kernel_dim (`int`, *optional*, defaults to 4):
+ Kernel size of the convolution used in linear attention layers.
+ linear_key_head_dim (`int`, *optional*, defaults to 128):
+ Dimension of each key head in linear attention.
+ linear_value_head_dim (`int`, *optional*, defaults to 128):
+ Dimension of each value head in linear attention.
+ linear_num_key_heads (`int`, *optional*, defaults to 16):
+ Number of key heads used in linear attention layers.
+ linear_num_value_heads (`int`, *optional*, defaults to 32):
+ Number of value heads used in linear attention layers.
+ decoder_sparse_step (`int`, *optional*, defaults to 1):
+ The frequency of the MoE layer.
+ moe_intermediate_size (`int`, *optional*, defaults to 512):
+ Intermediate size of the routed expert.
+ shared_expert_intermediate_size (`int`, *optional*, defaults to 512):
+ Intermediate size of the shared expert.
+ num_experts_per_tok (`int`, *optional*, defaults to 10):
+ Number of selected experts.
+ num_experts (`int`, *optional*, defaults to 512):
+ Number of routed experts.
+ norm_topk_prob (`bool`, *optional*, defaults to `True`):
+ Whether to normalize the topk probabilities.
+ output_router_logits (`bool`, *optional*, defaults to `False`):
+ Whether or not the router logits should be returned by the model. Enabling this will also
+ allow the model to output the auxiliary loss, including load balancing loss and router z-loss.
+ router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
+ The aux loss factor for the total loss.
+ mlp_only_layers (`list[int]`, *optional*, defaults to `[]`):
+ Indicate which layers use Qwen3NextMLP rather than Qwen3NextSparseMoeBlock
+ The list contains layer index, from 0 to num_layers-1 if we have num_layers layers
+ If `mlp_only_layers` is empty, `decoder_sparse_step` is used to determine the sparsity.
+ layer_types (`list[str]`, *optional*):
+ Types of each layer (attention or linear).
+ enable_qk_norm (`bool`, *optional*, defaults to `False`):
+ Whether to apply L2 normalization to the query and key embeddings.
+ router_bias (`bool`, *optional*, defaults to `False`):
+ Whether to use a bias in the router logits.
+ moe_router_score_function (`str`, *optional*, defaults to `"softmax"`):
+ The score function used in the MoE router.
+ ffn_token_shift (`str`, *optional*): Token shift before FFN/MoE. `None`, `"cat"`, or `"conv"`.
+ ffn_intermediate_token_shift (`str`, *optional*): Token shift in MLP before down_proj. `None`, `"cat"`, or `"conv"`.
+ attn_token_shift (`str`, *optional*): Token shift before attention. `None`, `"cat"`, or `"conv"`.
+ attn_q_token_shift (`str`, *optional*): Token shift on query after projection. `None`, `"cat"`, or `"conv"`.
+ attn_k_token_shift (`str`, *optional*): Token shift on key after projection. `None`, `"cat"`, or `"conv"`.
+ attn_v_token_shift (`str`, *optional*): Token shift on value after projection. `None`, `"cat"`, or `"conv"`.
+ token_shift_conv_size (`int`, *optional*, defaults to 4): Kernel size for token-shift Conv1d.
+ token_shift_conv_init (`str`, *optional*, defaults to `"default"`): Init for token-shift Conv1d; `"identity"` for causal identity.
+ ```python
+ >>> from transformers import Qwen3NextModel, Qwen3NextConfig
+
+ >>> # Initializing a Qwen3Next style configuration
+ >>> configuration = Qwen3NextConfig()
+
+ >>> # Initializing a model from the Qwen3-Next-80B-A3B style configuration
+ >>> model = Qwen3NextModel(configuration)
+
+ >>> # Accessing the model configuration
+ >>> configuration = model.config
+ ```
+ """
+
+ model_type = "qwen3_next"
+ keys_to_ignore_at_inference = ["past_key_values"]
+
+ base_model_tp_plan = {
+ "layers.*.self_attn.q_proj": "colwise",
+ "layers.*.self_attn.k_proj": "colwise",
+ "layers.*.self_attn.v_proj": "colwise",
+ "layers.*.self_attn.o_proj": "rowwise",
+ "layers.*.mlp.experts.*.gate_proj": "colwise",
+ "layers.*.mlp.experts.*.up_proj": "colwise",
+ "layers.*.mlp.experts.*.down_proj": "rowwise",
+ "layers.*.mlp.shared_experts.gate_proj": "colwise",
+ "layers.*.mlp.shared_experts.up_proj": "colwise",
+ "layers.*.mlp.shared_experts.down_proj": "rowwise",
+ "layers.*.mlp.gate_proj": "colwise",
+ "layers.*.mlp.up_proj": "colwise",
+ "layers.*.mlp.down_proj": "rowwise",
+ }
+ base_model_pp_plan = {
+ "embed_tokens": (["input_ids"], ["inputs_embeds"]),
+ "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
+ "norm": (["hidden_states"], ["hidden_states"]),
+ }
+
+ def __init__(
+ self,
+ vocab_size=151936,
+ hidden_size=2048,
+ intermediate_size=5632,
+ num_hidden_layers=48,
+ num_attention_heads=16,
+ num_key_value_heads=2,
+ hidden_act="silu",
+ max_position_embeddings=32768,
+ initializer_range=0.02,
+ rms_norm_eps=1e-6,
+ use_cache=True,
+ tie_word_embeddings=False,
+ rope_theta=10000.0,
+ rope_scaling=None,
+ partial_rotary_factor=0.25,
+ attention_bias=False,
+ attention_dropout=0.0,
+ head_dim=256,
+ linear_conv_kernel_dim=4,
+ linear_key_head_dim=128,
+ linear_value_head_dim=128,
+ linear_num_key_heads=16,
+ linear_num_value_heads=32,
+ decoder_sparse_step=1,
+ moe_intermediate_size=512,
+ shared_expert_intermediate_size=512,
+ num_experts_per_tok=10,
+ num_experts=512,
+ norm_topk_prob=True,
+ output_router_logits=False,
+ router_aux_loss_coef=0.001,
+ mlp_only_layers=[],
+ layer_types=None,
+ enable_qk_norm=False, # @o2iginal
+ router_bias=False, # @o2iginal
+ attn_output_gate=False, # @o2iginal
+ moe_router_score_function="softmax", # @xcx
+ # Cannon layer / token shifting (align with Megatron) @o2iginal
+ ffn_token_shift=None,
+ ffn_intermediate_token_shift=None,
+ attn_token_shift=None,
+ attn_q_token_shift=None,
+ attn_k_token_shift=None,
+ attn_v_token_shift=None,
+ token_shift_conv_size=4,
+ token_shift_conv_init="default",
+ # Separate RoPE for attention vs linear/RNN (GDN): "rope" or "nope"
+ attn_position_embedding_type="rope",
+ rnn_position_embedding_type="nope",
+ # Optional logits scaling for length extrapolation (attention only): None, float, or "log" / "log "
+ attn_logits_scaling=None,
+ **kwargs,
+ ):
+ super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
+ assert attn_position_embedding_type in ("rope", "nope"), (
+ f"attn_position_embedding_type must be 'rope' or 'nope', got {attn_position_embedding_type}"
+ )
+ assert rnn_position_embedding_type in ("rope", "nope"), (
+ f"rnn_position_embedding_type must be 'rope' or 'nope', got {rnn_position_embedding_type}"
+ )
+ self.attn_position_embedding_type = attn_position_embedding_type
+ self.rnn_position_embedding_type = rnn_position_embedding_type
+ self.attn_logits_scaling = attn_logits_scaling
+ self.vocab_size = vocab_size
+ self.max_position_embeddings = max_position_embeddings
+ self.hidden_size = hidden_size
+ self.intermediate_size = intermediate_size
+ self.num_hidden_layers = num_hidden_layers
+ self.num_attention_heads = num_attention_heads
+ self.num_key_value_heads = num_key_value_heads
+ self.hidden_act = hidden_act
+ self.initializer_range = initializer_range
+ self.rms_norm_eps = rms_norm_eps
+ self.use_cache = use_cache
+ self.rope_theta = rope_theta
+ self.rope_scaling = rope_scaling
+ self.partial_rotary_factor = partial_rotary_factor
+ self.attention_bias = attention_bias
+ self.attention_dropout = attention_dropout
+ self.head_dim = head_dim
+ rope_config_validation(self)
+
+ self.layer_types = layer_types
+ if self.layer_types is None:
+ interval_pattern = kwargs.get("full_attention_interval", 4)
+ self.layer_types = [
+ "linear_attention" if bool((i + 1) % interval_pattern) else "full_attention"
+ for i in range(self.num_hidden_layers)
+ ]
+ layer_type_validation(self.layer_types)
+
+ # linear attention part
+ self.linear_conv_kernel_dim = linear_conv_kernel_dim
+ self.linear_key_head_dim = linear_key_head_dim
+ self.linear_value_head_dim = linear_value_head_dim
+ self.linear_num_key_heads = linear_num_key_heads
+ self.linear_num_value_heads = linear_num_value_heads
+
+ # MoE arguments
+ self.decoder_sparse_step = decoder_sparse_step
+ self.moe_intermediate_size = moe_intermediate_size
+ self.shared_expert_intermediate_size = shared_expert_intermediate_size
+ self.num_experts_per_tok = num_experts_per_tok
+ self.num_experts = num_experts
+ self.norm_topk_prob = norm_topk_prob
+ self.output_router_logits = output_router_logits
+ self.router_aux_loss_coef = router_aux_loss_coef
+ self.mlp_only_layers = mlp_only_layers
+ self.enable_qk_norm = enable_qk_norm
+ self.router_bias = router_bias
+ self.attn_output_gate = attn_output_gate
+ self.moe_router_score_function = moe_router_score_function
+ self.enable_qk_norm = enable_qk_norm
+ self.router_bias = router_bias
+ self.attn_output_gate = attn_output_gate
+ self.moe_router_score_function = moe_router_score_function
+
+ # Token shifting (cannon layer): None | "cat" | "conv"
+ self.ffn_token_shift = ffn_token_shift
+ self.ffn_intermediate_token_shift = ffn_intermediate_token_shift
+ self.attn_token_shift = attn_token_shift
+ self.attn_q_token_shift = attn_q_token_shift
+ self.attn_k_token_shift = attn_k_token_shift
+ self.attn_v_token_shift = attn_v_token_shift
+ self.token_shift_conv_size = token_shift_conv_size
+ self.token_shift_conv_init = token_shift_conv_init
+
+__all__ = ["Qwen3NextConfig"]
diff --git a/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/hf2mcore.log b/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/hf2mcore.log
new file mode 100644
index 0000000000000000000000000000000000000000..ade7dec2973025c61bf98e7d153e30cd368c88e1
--- /dev/null
+++ b/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/hf2mcore.log
@@ -0,0 +1,1277 @@
+torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 42639 /mnt/ssd/lvzhihao/PostTrain/YuLan-Pretrain/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/cache_tmp/tmp/tmp.9kVbw0fz8t --hf-dir /mnt/ssd/cache_tmp/tmp/tmp.9kVbw0fz8t --mcore2hf --use-gpu --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 490000 --attention-backend auto --position-embedding-type rope --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 1920 --ffn-hidden-size 4800 --num-attention-heads 30 --untie-embeddings-and-output-weights --rotary-base 490000 --rotary-percent 1.00 --num-query-groups 6 --normalization RMSNorm --norm-epsilon 1e-6 --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/hdd/lvzhihao/mcore_models/grid_search_0318/checkpoint/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50 --save-dir /mnt/hdd/lvzhihao/mcore_models/grid_search_0318/checkpoint/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/iter_953-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --padded-vocab-size 99000 --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --debug --max-shard-size 20GB
+W0320 09:19:34.443000 3250 .venv/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
+W0320 09:19:34.443000 3250 .venv/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures.
+fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function.
+Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience.
+Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized
+using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
+Number of virtual stages per pipeline stage: None
+accumulate and all-reduce gradients in fp32 for bfloat16 data type.
+using torch.bfloat16 for parameters ...
+------------------------ arguments ------------------------
+ account_for_embedding_in_pipeline_split ......... False
+ account_for_loss_in_pipeline_split .............. False
+ accumulate_allreduce_grads_in_fp32 .............. True
+ activation_func_clamp_value ..................... None
+ adam_beta1 ...................................... 0.9
+ adam_beta2 ...................................... 0.95
+ adam_eps ........................................ 1e-08
+ adamw_lr_mup_scaler ............................. False
+ add_bias_linear ................................. False
+ add_position_embedding .......................... True
+ add_qkv_bias .................................... True
+ adlr_autoresume ................................. False
+ adlr_autoresume_interval ........................ 1000
+ align_grad_reduce ............................... True
+ align_param_gather .............................. False
+ allow_ambiguous_pad_tokens ...................... False
+ app_tag_run_name ................................ None
+ app_tag_run_version ............................. 0.0.0
+ apply_layernorm_1p .............................. False
+ apply_query_key_layer_scaling ................... False
+ apply_residual_connection_post_layernorm ........ False
+ apply_rope_fusion ............................... True
+ async_save ...................................... None
+ async_tensor_model_parallel_allreduce ........... True
+ attention_backend ............................... AttnBackend.auto
+ attention_dropout ............................... 0.1
+ attention_output_gate ........................... False
+ attention_softmax_in_fp32 ....................... False
+ attn_k_token_shift .............................. None
+ attn_output_gate ................................ None
+ attn_output_gate_rand_init ...................... False
+ attn_q_token_shift .............................. None
+ attn_token_shift ................................ None
+ attn_v_token_shift .............................. None
+ auto_detect_ckpt_format ......................... False
+ auto_generate_cu_seqlens ........................ False
+ auto_model ...................................... AutoModelForCausalLM
+ barrier_with_L1_time ............................ True
+ benchmark_eval .................................. False
+ benchmark_global_batch .......................... None
+ benchmark_interval .............................. None
+ benchmark_micro_batch ........................... None
+ benchmark_sequence_length ....................... None
+ benchmark_tasks ................................. None
+ bert_binary_head ................................ True
+ bert_embedder_type .............................. megatron
+ bert_load ....................................... None
+ bf16 ............................................ True
+ bias_dropout_fusion ............................. True
+ bias_gelu_fusion ................................ False
+ bias_swiglu_fusion .............................. True
+ biencoder_projection_dim ........................ 0
+ biencoder_shared_query_context_model ............ False
+ block_data_path ................................. None
+ cache_mla_latents ............................... False
+ calc_ft_timeouts ................................ False
+ calculate_per_token_loss ........................ False
+ check_for_large_grads ........................... False
+ check_for_nan_in_loss_and_grad .................. True
+ check_for_spiky_loss ............................ False
+ check_weight_hash_across_dp_replicas_interval ... None
+ ckpt_assume_constant_structure .................. False
+ ckpt_convert_format ............................. None
+ ckpt_convert_save ............................... None
+ ckpt_convert_update_legacy_dist_opt_format ...... False
+ ckpt_format ..................................... torch_dist
+ ckpt_fully_parallel_load ........................ False
+ ckpt_fully_parallel_save ........................ True
+ ckpt_fully_parallel_save_deprecated ............. False
+ ckpt_step ....................................... None
+ classes_fraction ................................ 1.0
+ clip_grad ....................................... 1.0
+ clone_scatter_output_in_embedding ............... True
+ config_logger_dir ...............................
+ consumed_train_samples .......................... 0
+ consumed_valid_samples .......................... 0
+ context_parallel_size ........................... 1
+ cp_comm_type .................................... ['p2p']
+ create_attention_mask_in_dataloader ............. True
+ cross_entropy_fusion_impl ....................... native
+ cross_entropy_loss_fusion ....................... False
+ cuda_graph_impl ................................. none
+ cuda_graph_scope ................................ []
+ cuda_graph_warmup_steps ......................... 3
+ data_args_path .................................. None
+ data_cache_path ................................. None
+ data_parallel_random_init ....................... False
+ data_parallel_sharding_strategy ................. no_shard
+ data_parallel_size .............................. 1
+ data_path ....................................... None
+ data_per_class_fraction ......................... 1.0
+ data_sharding ................................... True
+ dataloader_type ................................. single
+ ddp_average_in_collective ....................... False
+ ddp_bucket_size ................................. None
+ ddp_num_buckets ................................. None
+ ddp_pad_buckets_for_high_nccl_busbw ............. False
+ debug ........................................... True
+ decode_only_cuda_graphs ......................... False
+ decoder_first_pipeline_num_layers ............... None
+ decoder_last_pipeline_num_layers ................ None
+ decoder_num_layers .............................. None
+ decoder_seq_length .............................. None
+ decoupled_lr .................................... None
+ decoupled_min_lr ................................ None
+ decrease_batch_size_if_needed ................... False
+ defer_embedding_wgrad_compute ................... False
+ delay_wgrad_compute ............................. False
+ deprecated_use_mcore_models ..................... False
+ deterministic_mode .............................. False
+ dino_bottleneck_size ............................ 256
+ dino_freeze_last_layer .......................... 1
+ dino_head_hidden_size ........................... 2048
+ dino_local_crops_number ......................... 10
+ dino_local_img_size ............................. 96
+ dino_norm_last_layer ............................ False
+ dino_teacher_temp ............................... 0.07
+ dino_warmup_teacher_temp ........................ 0.04
+ dino_warmup_teacher_temp_epochs ................. 30
+ disable_attn_output_gate ........................ False
+ disable_bf16_reduced_precision_matmul ........... False
+ disable_chunked_prefill ......................... False
+ disable_explicit_attention_mask ................. False
+ disable_mamba_mem_eff_path ...................... False
+ disable_straggler_on_startup .................... False
+ disable_symmetric_registration .................. False
+ dist_ckpt_format_deprecated ..................... None
+ dist_ckpt_optim_fully_reshardable ............... True
+ dist_ckpt_save_pre_mcore_014 .................... False
+ dist_ckpt_strictness ............................ assume_ok_unexpected
+ distrib_optim_fully_reshardable_mem_efficient ... False
+ distribute_saved_activations .................... False
+ distributed_backend ............................. nccl
+ distributed_timeout_minutes ..................... 10
+ distributed_timeout_seconds_after_init .......... None
+ document_packing_algorithm ...................... random
+ dryrun .......................................... False
+ dump_param_to_param_group_map ................... None
+ emb_deviation_loss_coeff ........................ 0
+ emb_deviation_type .............................. None
+ embedding_init_method_std ....................... None
+ embedding_path .................................. None
+ empty_unused_memory_level ....................... 0
+ enable_cuda_graph ............................... False
+ enable_debug_logging ............................ False
+ enable_experimental ............................. False
+ enable_ft_package ............................... False
+ enable_full_sharding_in_hsdp .................... False
+ enable_gloo_process_groups ...................... True
+ enable_msc ...................................... True
+ enable_one_logger ............................... True
+ encoder_num_layers .............................. 56
+ encoder_seq_length .............................. 1
+ end_weight_decay ................................ 0.1
+ eod_mask_loss ................................... False
+ error_injection_rate ............................ 0
+ error_injection_type ............................ transient_error
+ eval_interval ................................... 1000
+ eval_iters ...................................... 10
+ evidence_data_path .............................. None
+ exit_duration_in_mins ........................... None
+ exit_interval ................................... None
+ exit_on_missing_checkpoint ...................... False
+ exit_signal_handler ............................. False
+ exp_avg_dtype ................................... torch.float32
+ exp_avg_sq_dtype ................................ torch.float32
+ expert_model_parallel_size ...................... 1
+ expert_tensor_parallel_size ..................... 1
+ external_cuda_graph ............................. False
+ ffn_hidden_size ................................. 4800
+ ffn_intermediate_token_shift .................... None
+ ffn_token_shift ................................. None
+ fine_grained_activation_offloading .............. False
+ finetune ........................................ False
+ first_last_layers_bf16 .......................... False
+ flash_decode .................................... False
+ fp16 ............................................ False
+ fp16_lm_cross_entropy ........................... False
+ fp32_residual_connection ........................ False
+ fp4 ............................................. None
+ fp4_param ....................................... False
+ fp4_recipe ...................................... nvfp4
+ fp8 ............................................. None
+ fp8_amax_compute_algo ........................... most_recent
+ fp8_amax_history_len ............................ 1
+ fp8_interval .................................... 1
+ fp8_margin ...................................... 0
+ fp8_param_gather ................................ False
+ fp8_recipe ...................................... delayed
+ fp8_wgrad ....................................... True
+ freeze_layernorm_weight ......................... False
+ freeze_non_mamba ................................ False
+ fsdp_double_buffer .............................. False
+ full_validation ................................. False
+ gdn_cp_impl ..................................... cp2hp
+ geglu ........................................... False
+ global_batch_size ............................... 1024
+ glu_linear_offset ............................... 0.0
+ grad_reduce_in_bf16 ............................. False
+ gradient_accumulation_fusion .................... True
+ gradient_reduce_div_fusion ...................... True
+ group_query_attention ........................... True
+ grpo_clamp_eps_lower ............................ 0.01
+ grpo_clamp_eps_upper ............................ 0.01
+ grpo_default_temperature ........................ 1.0
+ grpo_default_top_p .............................. 0
+ grpo_entropy_term_weight ........................ 0.0
+ grpo_filter_groups_with_same_reward ............. False
+ grpo_group_size ................................. 2
+ grpo_iterations ................................. 2
+ grpo_kl_beta .................................... 0.001
+ grpo_prompts_per_step ........................... 32
+ head_lr_mult .................................... 1.0
+ heterogeneous_layers_config_encoded_json ........ None
+ heterogeneous_layers_config_path ................ None
+ hf_dir .......................................... /mnt/ssd/cache_tmp/tmp/tmp.9kVbw0fz8t
+ hidden_dropout .................................. 0.1
+ hidden_size ..................................... 1920
+ hierarchical_context_parallel_sizes ............. None
+ high_priority_stream_groups ..................... []
+ hybrid_attention_ratio .......................... 0.0
+ hybrid_context_parallel ......................... False
+ hybrid_mlp_ratio ................................ 0.0
+ hybrid_override_pattern ......................... None
+ hysteresis ...................................... 2
+ ict_head_size ................................... None
+ ict_load ........................................ None
+ img_h ........................................... 224
+ img_w ........................................... 224
+ increase_log_level_interval ..................... 1000
+ increase_log_level_iters ........................ 5
+ indexer_batch_size .............................. 128
+ indexer_log_interval ............................ 1000
+ inference_batch_times_seqlen_threshold .......... -1
+ inference_dynamic_batching ...................... False
+ inference_dynamic_batching_block_size ........... 256
+ inference_dynamic_batching_buffer_guaranteed_fraction 0.2
+ inference_dynamic_batching_buffer_overflow_factor None
+ inference_dynamic_batching_buffer_size_gb ....... 40.0
+ inference_dynamic_batching_max_requests_override None
+ inference_dynamic_batching_max_tokens_override .. None
+ inference_dynamic_batching_num_cuda_graphs ...... 16
+ inference_dynamic_batching_track_paused_request_events False
+ inference_dynamic_batching_unified_memory_level . 0
+ inference_max_batch_size ........................ 8
+ inference_max_seq_length ........................ 2560
+ inference_rng_tracker ........................... False
+ init_method_std ................................. 0.006
+ init_method_xavier_uniform ...................... False
+ init_model_with_meta_device ..................... False
+ initial_loss_scale .............................. 4294967296
+ inprocess_active_world_size ..................... 1
+ inprocess_barrier_timeout ....................... 120
+ inprocess_completion_timeout .................... 120
+ inprocess_empty_cuda_cache ...................... False
+ inprocess_granularity ........................... node
+ inprocess_hard_timeout .......................... 90
+ inprocess_heartbeat_interval .................... 30
+ inprocess_heartbeat_timeout ..................... 60
+ inprocess_last_call_wait ........................ 1
+ inprocess_max_iterations ........................ None
+ inprocess_monitor_process_interval .............. 1.0
+ inprocess_monitor_thread_interval ............... 1.0
+ inprocess_progress_watchdog_interval ............ 1.0
+ inprocess_restart ............................... False
+ inprocess_soft_timeout .......................... 60
+ inprocess_termination_grace_time ................ 1
+ is_hybrid_model ................................. False
+ iter_per_epoch .................................. 1250
+ iterations_to_skip .............................. []
+ keep_fp8_transpose_cache ........................ False
+ kitchen_config_file ............................. None
+ kitchen_recipe_number ........................... None
+ kv_channels ..................................... 64
+ kv_lora_rank .................................... 32
+ langrl_env_config ............................... None
+ langrl_external_server .......................... False
+ langrl_inference_server_conversation_template ... None
+ langrl_inference_server_type .................... inplace_megatron
+ lazy_mpu_init ................................... None
+ legacy_tokenizer ................................ False
+ linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1]
+ linear_attention_type ........................... gated_delta_net
+ linear_conv_kernel_dim .......................... 4
+ linear_key_head_dim ............................. 64
+ linear_num_key_heads ............................ 8
+ linear_num_value_heads .......................... 32
+ linear_value_head_dim ........................... 64
+ load ............................................ None
+ load_complemental_dataset ....................... None
+ load_dir ........................................ /mnt/hdd/lvzhihao/mcore_models/grid_search_0318/checkpoint/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50
+ load_main_params_from_ckpt ...................... None
+ local_rank ...................................... 0
+ log_energy ...................................... False
+ log_hidden_states ............................... []
+ log_interval .................................... 100
+ log_loss_scale_to_tensorboard ................... True
+ log_memory_to_tensorboard ....................... False
+ log_num_zeros_in_grad ........................... False
+ log_params ...................................... []
+ log_params_norm ................................. False
+ log_per_module_grad_rms ......................... False
+ log_per_module_update_rms ....................... False
+ log_progress .................................... False
+ log_straggler ................................... False
+ log_throughput .................................. False
+ log_timers_to_tensorboard ....................... False
+ log_validation_ppl_to_tensorboard ............... False
+ log_world_size_to_tensorboard ................... False
+ logging_level ................................... 1
+ loss_scale ...................................... None
+ loss_scale_window ............................... 1000
+ lr .............................................. 2e-05
+ lr_decay_iters .................................. 430000
+ lr_decay_samples ................................ None
+ lr_decay_style .................................. cosine
+ lr_warmup_fraction .............................. 0.001
+ lr_warmup_init .................................. 0.0
+ lr_warmup_iters ................................. 0
+ lr_warmup_samples ............................... 0
+ lr_wsd_decay_iters .............................. None
+ lr_wsd_decay_samples ............................ None
+ lr_wsd_decay_style .............................. exponential
+ main_grads_dtype ................................ torch.float32
+ main_params_dtype ............................... torch.float32
+ make_vocab_size_divisible_by .................... 128
+ mamba_disable_cp ................................ False
+ mamba_expand .................................... 2
+ mamba_head_dim .................................. 64
+ mamba_num_groups ................................ 8
+ mamba_num_heads ................................. None
+ mamba_state_dim ................................. 128
+ manual_gc ....................................... False
+ manual_gc_eval .................................. True
+ manual_gc_interval .............................. 0
+ mask_factor ..................................... 1.0
+ mask_prob ....................................... 0.15
+ mask_type ....................................... random
+ masked_softmax_fusion ........................... True
+ max_position_embeddings ......................... 490000
+ max_seqlen_per_cp_rank .......................... None
+ max_shard_size .................................. 20GB
+ max_tokens_to_oom ............................... 12000
+ mcore2hf ........................................ True
+ memory_snapshot_path ............................ None
+ merge_file ...................................... None
+ micro_batch_size ................................ 1
+ microbatch_group_size_per_vp_stage .............. None
+ mid_level_dataset_surplus ....................... 0.005
+ min_loss_scale .................................. 1.0
+ min_lr .......................................... 6e-06
+ min_offloaded_tensor_size ....................... 1048576
+ mlp_chunks_for_prefill .......................... 1
+ mmap_bin_files .................................. True
+ mock_data ....................................... False
+ model_type ...................................... GPT
+ moe_apply_probs_on_input ........................ False
+ moe_aux_loss_coeff .............................. 0.0
+ moe_deepep_num_sms .............................. 20
+ moe_enable_deepep ............................... False
+ moe_expert_capacity_factor ...................... None
+ moe_extended_tp ................................. False
+ moe_ffn_hidden_size ............................. None
+ moe_flex_dispatcher_backend ..................... deepep
+ moe_grouped_gemm ................................ False
+ moe_hybridep_num_sms ............................ 16
+ moe_input_jitter_eps ............................ None
+ moe_layer_freq .................................. 1
+ moe_layer_recompute ............................. False
+ moe_pad_expert_input_to_capacity ................ False
+ moe_pad_experts_for_cuda_graph_inference ........ False
+ moe_per_layer_logging ........................... False
+ moe_permute_fusion .............................. False
+ moe_router_bias_update_method ................... sign
+ moe_router_bias_update_rate ..................... 0.001
+ moe_router_dtype ................................ None
+ moe_router_enable_expert_bias ................... False
+ moe_router_force_load_balancing ................. False
+ moe_router_fusion ............................... False
+ moe_router_group_topk ........................... None
+ moe_router_load_balancing_type .................. aux_loss
+ moe_router_num_groups ........................... None
+ moe_router_padding_for_fp8 ...................... False
+ moe_router_padding_for_quantization ............. False
+ moe_router_pre_softmax .......................... False
+ moe_router_score_function ....................... softmax
+ moe_router_topk ................................. 2
+ moe_router_topk_scaling_factor .................. None
+ moe_shared_expert_gate .......................... False
+ moe_shared_expert_intermediate_size ............. None
+ moe_shared_expert_overlap ....................... False
+ moe_token_dispatcher_type ....................... allgather
+ moe_token_drop_policy ........................... probs
+ moe_upcycling_granularity ....................... 1
+ moe_use_legacy_grouped_gemm ..................... False
+ moe_use_upcycling ............................... False
+ moe_z_loss_coeff ................................ None
+ mrope_section ................................... None
+ mscale .......................................... 1.0
+ mscale_all_dim .................................. 0.0
+ mtp_linear_attention_type ....................... None
+ mtp_loss_scaling_factor ......................... 0.1
+ mtp_num_layers .................................. None
+ multi_latent_attention .......................... False
+ multiple_validation_sets ........................ False
+ muon_ball_momentum .............................. 0.9
+ muon_ball_msign_steps ........................... 5
+ muon_ball_power_iteration_steps ................. 10
+ muon_ball_qkv_split_mode ........................ component
+ muon_ball_radius_mode ........................... spectral_mup
+ muon_ball_retract_alpha ......................... 0.05
+ muon_ball_retract_mode .......................... hard
+ muon_ball_scale_mode ............................ spectral_mup
+ muon_ball_split_fc1 ............................. True
+ muon_ball_split_moe_experts ..................... True
+ muon_ball_split_qkv ............................. True
+ muon_ball_use_nesterov .......................... True
+ muon_extra_scale_factor ......................... 1.0
+ muon_fp32_matmul_prec ........................... medium
+ muon_momentum ................................... 0.9
+ muon_num_ns_steps ............................... 5
+ muon_qkv_split_mode ............................. component
+ muon_scale_mode ................................. spectral_mup
+ muon_scale_vectorized_mode ...................... full
+ muon_split_fc1 .................................. True
+ muon_split_moe_experts .......................... True
+ muon_split_qkv .................................. True
+ muon_tp_mode .................................... blockwise
+ muon_use_nesterov ............................... False
+ muon_vectorize .................................. []
+ muon_vectorize_attn_dim ......................... hidden_size
+ nccl_all_reduce_for_prefill ..................... False
+ nccl_communicator_config_path ................... None
+ nccl_ub ......................................... False
+ no_load_optim ................................... True
+ no_load_rng ..................................... True
+ no_load_scheduler ............................... None
+ no_persist_layer_norm ........................... False
+ no_rope_freq .................................... None
+ no_save_optim ................................... None
+ no_save_rng ..................................... None
+ no_save_step_one ................................ None
+ no_weight_decay_cond_type ....................... None
+ non_persistent_ckpt_type ........................ None
+ non_persistent_global_ckpt_dir .................. None
+ non_persistent_local_ckpt_algo .................. fully_parallel
+ non_persistent_local_ckpt_dir ................... None
+ non_persistent_save_interval .................... None
+ norm_epsilon .................................... 1e-06
+ normalization ................................... RMSNorm
+ num_attention_heads ............................. 30
+ num_channels .................................... 3
+ num_classes ..................................... 1000
+ num_dataset_builder_threads ..................... 1
+ num_distributed_optimizer_instances ............. 1
+ num_experts ..................................... None
+ num_hf_saver .................................... None
+ num_layers ...................................... 56
+ num_layers_at_end_in_bf16 ....................... 1
+ num_layers_at_start_in_bf16 ..................... 1
+ num_layers_per_virtual_pipeline_stage ........... None
+ num_query_groups ................................ 6
+ num_virtual_stages_per_pipeline_rank ............ None
+ num_workers ..................................... 2
+ object_storage_cache_path ....................... None
+ offload_modules ................................. []
+ one_logger_async ................................ False
+ one_logger_project .............................. megatron-lm
+ one_logger_run_name ............................. None
+ onnx_safe ....................................... None
+ openai_gelu ..................................... False
+ optimizer ....................................... adam
+ optimizer_cpu_offload ........................... False
+ optimizer_offload_fraction ...................... 1.0
+ output_bert_embeddings .......................... False
+ overlap_cpu_optimizer_d2h_h2d ................... False
+ overlap_grad_reduce ............................. False
+ overlap_moe_expert_parallel_comm ................ False
+ overlap_p2p_comm ................................ False
+ overlap_p2p_comm_warmup_flush ................... False
+ overlap_param_gather ............................ False
+ overlap_param_gather_with_optimizer_step ........ False
+ override_hf_eod_token_id ........................ None
+ override_opt_param_scheduler .................... False
+ padded_vocab_size ............................... 99000
+ params_dtype .................................... torch.bfloat16
+ patch_dim ....................................... 16
+ per_split_data_args_path ........................ None
+ perform_initialization .......................... True
+ perform_rl_step ................................. False
+ pin_cpu_grads ................................... True
+ pin_cpu_params .................................. True
+ pipeline_model_parallel_comm_backend ............ None
+ pipeline_model_parallel_layout .................. None
+ pipeline_model_parallel_size .................... 1
+ position_embedding_type ......................... rope
+ pretrain_script ................................. mcore_gdn_moe.model_provider
+ pretrained_checkpoint ........................... None
+ profile ......................................... False
+ profile_ranks ................................... [0]
+ profile_step_end ................................ 12
+ profile_step_start .............................. 10
+ q_lora_rank ..................................... None
+ qk_head_dim ..................................... 128
+ qk_l2_norm ...................................... False
+ qk_layernorm .................................... False
+ qk_pos_emb_head_dim ............................. 64
+ query_in_block_prob ............................. 0.1
+ quick_geglu ..................................... False
+ rampup_batch_size ............................... None
+ rank ............................................ 0
+ recompute_granularity ........................... None
+ recompute_method ................................ None
+ recompute_modules ............................... None
+ recompute_num_layers ............................ None
+ record_memory_history ........................... False
+ relative_attention_max_distance ................. 128
+ relative_attention_num_buckets .................. 32
+ reparam_checkpoint .............................. None
+ reparam_fallback_value .......................... None
+ reparam_keys .................................... None
+ replication ..................................... False
+ replication_factor .............................. 2
+ replication_jump ................................ None
+ rerun_mode ...................................... validate_results
+ reset_attention_mask ............................ False
+ reset_iteration_one_to_zero ..................... False
+ reset_position_ids .............................. False
+ result_rejected_tracker_filename ................ None
+ retriever_report_topk_accuracies ................ []
+ retriever_score_scaling ......................... False
+ retriever_seq_length ............................ 256
+ retro_add_retriever ............................. False
+ retro_attention_gate ............................ 1
+ retro_cyclic_train_iters ........................ None
+ retro_encoder_attention_dropout ................. 0.1
+ retro_encoder_hidden_dropout .................... 0.1
+ retro_encoder_layers ............................ 2
+ retro_num_neighbors ............................. 2
+ retro_num_retrieved_chunks ...................... 2
+ retro_project_dir ............................... None
+ retro_verify_neighbor_count ..................... True
+ reuse_grad_buf_for_mxfp8_param_ag ............... False
+ rl_calculate_intra_group_similarity ............. False
+ rl_importance_sampling_truncation_coef .......... None
+ rl_inference_logprobs_is_correction ............. False
+ rl_offload_kv_cache_during_training ............. False
+ rl_offload_optimizer_during_inference ........... False
+ rl_partial_rollouts ............................. False
+ rl_prompts_per_eval ............................. 32
+ rl_remove_kv_cache_during_training .............. False
+ rl_reset_cuda_graphs ............................ False
+ rl_sequence_packing_algo ........................ fifo
+ rl_sequence_packing_bin_size .................... 8192
+ rl_use_sequence_packing ......................... False
+ rope_scaling_factor ............................. 8.0
+ rope_type ....................................... None
+ rotary_base ..................................... 490000
+ rotary_interleaved .............................. False
+ rotary_percent .................................. 1.0
+ rotary_scaling_factor ........................... 1.0
+ rotary_seq_len_interpolation_factor ............. None
+ run_workload_inspector_server ................... False
+ sample_rate ..................................... 1.0
+ save ............................................ None
+ save_after_load ................................. False
+ save_dir ........................................ /mnt/hdd/lvzhihao/mcore_models/grid_search_0318/checkpoint/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/iter_953-hf
+ save_interval ................................... 10000
+ save_retain_interval ............................ None
+ scatter_gather_tensors_in_pipeline .............. True
+ seed ............................................ 1234
+ seq_length ...................................... 1
+ sequence_parallel ............................... False
+ sft ............................................. False
+ sft_tokenizer_prompt_format ..................... nemotron-h-aligned
+ sgd_momentum .................................... 0.9
+ sharp_enabled_group ............................. None
+ short_seq_prob .................................. 0.1
+ skip_train ...................................... True
+ skipped_train_samples ........................... 0
+ softmax_type .................................... vanilla
+ spec ............................................ None
+ spectral_ball_momentum .......................... 0.9
+ spectral_ball_msign_steps ....................... 8
+ spectral_ball_power_iteration_steps ............. 20
+ spectral_ball_qkv_split_mode .................... component
+ spectral_ball_radius_mode ....................... spectral_mup
+ spectral_ball_retract_alpha ..................... 0.05
+ spectral_ball_retract_mode ...................... hard
+ spectral_ball_scale_mode ........................ spectral_mup
+ spectral_ball_solver ............................ bisection
+ spectral_ball_solver_max_iterations ............. 20
+ spectral_ball_solver_tolerance_f ................ 1e-08
+ spectral_ball_split_fc1 ......................... True
+ spectral_ball_split_moe_experts ................. True
+ spectral_ball_split_qkv ......................... True
+ spectral_ball_use_nesterov ...................... True
+ spectral_mup_init ............................... False
+ split ........................................... None
+ split_expert_init ............................... True
+ split_fc1_init .................................. True
+ split_qkv_init .................................. True
+ split_qkv_init_mode ............................. group
+ sqreglu ......................................... False
+ squared_relu .................................... False
+ start_samples ................................... None
+ start_weight_decay .............................. 0.1
+ straggler_ctrlr_port ............................ 65535
+ straggler_minmax_count .......................... 1
+ strict_fsdp_dtensor_load ........................ True
+ suggested_communication_unit_size ............... None
+ swanlab_exp_name ................................
+ swanlab_project .................................
+ swanlab_save_dir ................................
+ swanlab_workspace ...............................
+ swiglu .......................................... True
+ swin_backbone_type .............................. tiny
+ symmetric_ar_type ............................... None
+ synchronizer .................................... mcore_gdn_moe
+ target_ckpt_format .............................. torch_dist
+ te_rng_tracker .................................. False
+ tensor_model_parallel_size ...................... 1
+ tensorboard_dir ................................. None
+ tensorboard_log_interval ........................ 1
+ tensorboard_queue_size .......................... 1000
+ test_data_path .................................. None
+ test_mode ....................................... False
+ tiktoken_num_special_tokens ..................... 1000
+ tiktoken_pattern ................................ None
+ tiktoken_special_tokens ......................... None
+ timing_log_level ................................ 0
+ timing_log_option ............................... minmax
+ titles_data_path ................................ None
+ token_shift_conv_init ........................... default
+ token_shift_conv_size ........................... 4
+ tokenizer_metadata .............................. None
+ tokenizer_model ................................. /mnt/ssd/cache_tmp/tmp/tmp.9kVbw0fz8t
+ tokenizer_type .................................. HuggingFaceTokenizer
+ torch_fsdp2_reshard_after_forward ............... True
+ tp_comm_bootstrap_backend ....................... nccl
+ tp_comm_bulk_dgrad .............................. True
+ tp_comm_bulk_wgrad .............................. True
+ tp_comm_overlap ................................. False
+ tp_comm_overlap_ag .............................. True
+ tp_comm_overlap_cfg ............................. None
+ tp_comm_overlap_rs .............................. True
+ tp_comm_overlap_rs_dgrad ........................ False
+ tp_comm_split_ag ................................ True
+ tp_comm_split_rs ................................ True
+ train_data_path ................................. None
+ train_iters ..................................... 500000
+ train_samples ................................... None
+ train_sync_interval ............................. None
+ transformer_impl ................................ transformer_engine
+ transformer_pipeline_model_parallel_size ........ 1
+ trust_remote_code ............................... False
+ untie_embeddings_and_output_weights ............. True
+ use_checkpoint_args ............................. False
+ use_checkpoint_opt_param_scheduler .............. False
+ use_cpu_initialization .......................... True
+ use_dist_ckpt ................................... True
+ use_dist_ckpt_deprecated ........................ False
+ use_distributed_optimizer ....................... False
+ use_flash_attn .................................. False
+ use_fused_weighted_squared_relu ................. False
+ use_gpu ......................................... True
+ use_legacy_models ............................... False
+ use_megatron_fsdp ............................... False
+ use_mp_args_from_checkpoint_args ................ False
+ use_one_sent_docs ............................... False
+ use_persistent_ckpt_worker ...................... False
+ use_precision_aware_optimizer ................... False
+ use_pytorch_profiler ............................ False
+ use_ring_exchange_p2p ........................... False
+ use_rope_scaling ................................ False
+ use_rotary_position_embeddings .................. False
+ use_sharp ....................................... False
+ use_te_activation_func .......................... False
+ use_tokenizer_model_from_checkpoint_args ........ True
+ use_torch_fsdp2 ................................. False
+ use_torch_optimizer_for_cpu_offload ............. False
+ use_tp_pp_dp_mapping ............................ False
+ v_head_dim ...................................... 128
+ valid_data_path ................................. None
+ variable_seq_lengths ............................ False
+ virtual_pipeline_model_parallel_size ............ None
+ vision_backbone_type ............................ vit
+ vision_pretraining .............................. False
+ vision_pretraining_type ......................... classify
+ vocab_extra_ids ................................. 0
+ vocab_file ...................................... None
+ vocab_size ...................................... None
+ wandb_entity ....................................
+ wandb_exp_name ..................................
+ wandb_project ...................................
+ wandb_save_dir ..................................
+ weight_decay .................................... 0.1
+ weight_decay_incr_style ......................... constant
+ wgrad_deferral_limit ............................ 0
+ window_attn_skip_freq ........................... None
+ window_size ..................................... None
+ word_embedding_dropout_prob ..................... 0.0
+ world_size ...................................... 1
+ yaml_cfg ........................................ None
+-------------------- end of arguments ---------------------
+INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024
+> building HuggingFaceTokenizer tokenizer ...
+You are using the default legacy behaviour of the ": 43,
+ "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "eos_token": {
+ "content": "",
+ "lstrip": false,
+ "normalized": false,
+ "rstrip": false,
+ "single_word": false
+ },
+ "pad_token": {
+ "content": "",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 2,
+ "content": "<|endoftext|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 102,
+ "content": "",
+ "type_id": 0
+ }
+ },
+ {
+ "Sequence": {
+ "id": "A",
+ "type_id": 0
+ }
+ }
+ ],
+ "pair": [
+ {
+ "SpecialToken": {
+ "id": "",
+ "type_id": 0
+ }
+ },
+ {
+ "Sequence": {
+ "id": "A",
+ "type_id": 0
+ }
+ },
+ {
+ "SpecialToken": {
+ "id": "",
+ "type_id": 1
+ }
+ },
+ {
+ "Sequence": {
+ "id": "B",
+ "type_id": 1
+ }
+ }
+ ],
+ "special_tokens": {
+ "": {
+ "id": "",
+ "ids": [
+ 1
+ ],
+ "tokens": [
+ ""
+ ]
+ }
+ }
+ },
+ "decoder": {
+ "type": "Sequence",
+ "decoders": [
+ {
+ "type": "Replace",
+ "pattern": {
+ "String": "▁"
+ },
+ "content": " "
+ },
+ {
+ "type": "ByteFallback"
+ },
+ {
+ "type": "Fuse"
+ },
+ {
+ "type": "Strip",
+ "content": " ",
+ "start": 1,
+ "stop": 0
+ }
+ ]
+ },
+ "model": {
+ "type": "BPE",
+ "dropout": null,
+ "unk_token": "": 1,
+ "<|endoftext|>": 2,
+ "": 7,
+ "
": 8,
+ "": 9,
+ "
": 10,
+ "": 11,
+ "
": 12,
+ "": 13,
+ " ": 14,
+ "": 15,
+ "": 16,
+ "": 17,
+ "": 18,
+ " ": 19,
+ "
": 21,
+ "": 20,
+ "
": 22,
+ " ": 23,
+ "": 24,
+ " ": 31,
+ "": 32,
+ "
": 33,
+ "
": 34,
+ "
": 35,
+ "": 36,
+ "": 37,
+ "": 38,
+ "
": 39,
+ "": 40,
+ "": 41,
+ "
": 42,
+ "