diff --git a/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/config.json b/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/config.json
deleted file mode 100644
index 424c56db43fd50dd677a438fbcc7fe52824159a7..0000000000000000000000000000000000000000
--- a/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/config.json
+++ /dev/null
@@ -1,66 +0,0 @@
-{
- "architectures": [
- "Qwen3NextForCausalLM"
- ],
- "attention_bias": true,
- "attention_dropout": 0.0,
- "attn_output_gate": false,
- "auto_map": {
- "AutoConfig": "configuration_qwen3_next.Qwen3NextConfig",
- "AutoModel": "modeling_qwen3_next.Qwen3NextForCausalLM",
- "AutoModelForCausalLM": "modeling_qwen3_next.Qwen3NextForCausalLM"
- },
- "bos_token_id": 1,
- "decoder_sparse_step": 1,
- "dtype": "float32",
- "enable_qk_norm": false,
- "eos_token_id": 2,
- "full_attention_interval": 0,
- "head_dim": 64,
- "hidden_act": "silu",
- "hidden_size": 1920,
- "initializer_range": 0.02,
- "intermediate_size": 4800,
- "layer_types": ["linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "full_attention", "linear_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "full_attention", "linear_attention", "full_attention", "full_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention", "linear_attention"],
- "linear_conv_kernel_dim": 4,
- "linear_key_head_dim": 64,
- "linear_num_key_heads": 8,
- "linear_num_value_heads": 32,
- "linear_value_head_dim": 64,
- "max_position_embeddings": 32768,
- "mlp_only_layers": [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55],
- "num_experts_per_tok": 2,
- "num_experts": 0,
- "model_type": "qwen3_next",
- "moe_intermediate_size": 0,
- "norm_topk_prob": true,
- "num_attention_heads": 30,
- "num_hidden_layers": 56,
- "num_key_value_heads": 6,
- "output_router_logits": false,
- "partial_rotary_factor": 1.0,
- "rms_norm_eps": 1e-06,
- "rope_scaling": null,
- "rope_theta": 490000,
- "router_aux_loss_coef": 0.001,
- "router_bias": false,
- "moe_router_score_function": "softmax",
- "shared_expert_intermediate_size": 0,
- "use_shared_expert_gate": true,
- "tie_word_embeddings": false,
- "transformers_version": "4.57.1",
- "use_cache": true,
- "use_sliding_window": false,
- "ffn_token_shift": null,
- "ffn_intermediate_token_shift": null,
- "attn_token_shift": null,
- "attn_q_token_shift": null,
- "attn_k_token_shift": null,
- "attn_v_token_shift": null,
- "token_shift_conv_size": 4,
- "token_shift_conv_init": "default",
- "attn_position_embedding_type": "rope",
- "rnn_position_embedding_type": "nope",
- "attn_logits_scaling": null,
- "vocab_size": 99000
-}
\ No newline at end of file
diff --git a/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/configuration_qwen3_next.py b/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/configuration_qwen3_next.py
deleted file mode 100644
index 7f41fdfa8d7f284394825ebb29537c2b902a5304..0000000000000000000000000000000000000000
--- a/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/configuration_qwen3_next.py
+++ /dev/null
@@ -1,329 +0,0 @@
-# coding=utf-8
-# Copyright 2025 The Qwen team, Alibaba Group and the HuggingFace Inc. team. All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-"""Qwen3-Next model configuration"""
-
-from transformers.configuration_utils import PretrainedConfig, layer_type_validation
-from transformers.modeling_rope_utils import rope_config_validation
-from transformers.utils import logging
-
-
-logger = logging.get_logger(__name__)
-
-
-class Qwen3NextConfig(PretrainedConfig):
- r"""
- This is the configuration class to store the configuration of a [`Qwen3NextModel`]. It is used to instantiate a
- Qwen3-Next model according to the specified arguments, defining the model architecture.
- Instantiating a configuration with the defaults will yield a similar configuration to that of
- Qwen3-Next-80B-A3B-Instruct [Qwen/Qwen3-Next-80B-A3B-Instruct](https://huggingface.co/Qwen/Qwen3-Next-80B-A3B-Instruct).
-
- Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
- documentation from [`PretrainedConfig`] for more information.
-
-
- Args:
- vocab_size (`int`, *optional*, defaults to 151936):
- Vocabulary size of the model. Defines the number of different tokens that can be represented by the
- `inputs_ids`.
- hidden_size (`int`, *optional*, defaults to 2048):
- Dimension of the hidden representations.
- intermediate_size (`int`, *optional*, defaults to 5632):
- Dimension of the MLP representations.
- num_hidden_layers (`int`, *optional*, defaults to 48):
- Number of hidden layers in the Transformer encoder.
- num_attention_heads (`int`, *optional*, defaults to 16):
- Number of attention heads for each attention layer in the Transformer encoder.
- num_key_value_heads (`int`, *optional*, defaults to 2):
- This is the number of key_value heads that should be used to implement Grouped Query Attention. If
- `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
- `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
- converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
- by meanpooling all the original heads within that group. For more details checkout [this
- paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to `32`.
- hidden_act (`str`, *optional*, defaults to `"silu"`):
- The non-linear activation function in the decoder.
- max_position_embeddings (`int`, *optional*, defaults to 32768):
- The maximum sequence length that this model might ever be used with.
- initializer_range (`float`, *optional*, defaults to 0.02):
- The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
- rms_norm_eps (`float`, *optional*, defaults to 1e-06):
- The epsilon used by the rms normalization layers.
- use_cache (`bool`, *optional*, defaults to `True`):
- Whether or not the model should return the last key/values attentions (not used by all models). Only
- relevant if `config.is_decoder=True`.
- tie_word_embeddings (`bool`, *optional*, defaults to `False`):
- Whether the model's input and output word embeddings should be tied.
- rope_theta (`float`, *optional*, defaults to 10000.0):
- The base period of the RoPE embeddings.
- rope_scaling (`Dict`, *optional*):
- Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
- and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
- accordingly.
- Expected contents:
- `rope_type` (`str`):
- The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
- 'llama3'], with 'default' being the original RoPE implementation.
- `factor` (`float`, *optional*):
- Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
- most scaling types, a `factor` of x will enable the model to handle sequences of length x *
- original maximum pre-trained length.
- `original_max_position_embeddings` (`int`, *optional*):
- Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
- pretraining.
- `attention_factor` (`float`, *optional*):
- Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
- computation. If unspecified, it defaults to value recommended by the implementation, using the
- `factor` field to infer the suggested value.
- `beta_fast` (`float`, *optional*):
- Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
- ramp function. If unspecified, it defaults to 32.
- `beta_slow` (`float`, *optional*):
- Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
- ramp function. If unspecified, it defaults to 1.
- `short_factor` (`List[float]`, *optional*):
- Only used with 'longrope'. The scaling factor to be applied to short contexts (<
- `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
- size divided by the number of attention heads divided by 2
- `long_factor` (`List[float]`, *optional*):
- Only used with 'longrope'. The scaling factor to be applied to long contexts (<
- `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
- size divided by the number of attention heads divided by 2
- `low_freq_factor` (`float`, *optional*):
- Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
- `high_freq_factor` (`float`, *optional*):
- Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
- partial_rotary_factor (`float`, *optional*, defaults to 0.25):
- Percentage of the query and keys which will have rotary embedding.
- attention_bias (`bool`, *optional*, defaults to `False`):
- Whether to use a bias in the query, key, value and output projection layers during self-attention.
- attention_dropout (`float`, *optional*, defaults to 0.0):
- The dropout ratio for the attention probabilities.
- head_dim (`int`, *optional*, defaults to 256):
- Projection weights dimension in multi-head attention.
- linear_conv_kernel_dim (`int`, *optional*, defaults to 4):
- Kernel size of the convolution used in linear attention layers.
- linear_key_head_dim (`int`, *optional*, defaults to 128):
- Dimension of each key head in linear attention.
- linear_value_head_dim (`int`, *optional*, defaults to 128):
- Dimension of each value head in linear attention.
- linear_num_key_heads (`int`, *optional*, defaults to 16):
- Number of key heads used in linear attention layers.
- linear_num_value_heads (`int`, *optional*, defaults to 32):
- Number of value heads used in linear attention layers.
- decoder_sparse_step (`int`, *optional*, defaults to 1):
- The frequency of the MoE layer.
- moe_intermediate_size (`int`, *optional*, defaults to 512):
- Intermediate size of the routed expert.
- shared_expert_intermediate_size (`int`, *optional*, defaults to 512):
- Intermediate size of the shared expert.
- num_experts_per_tok (`int`, *optional*, defaults to 10):
- Number of selected experts.
- num_experts (`int`, *optional*, defaults to 512):
- Number of routed experts.
- norm_topk_prob (`bool`, *optional*, defaults to `True`):
- Whether to normalize the topk probabilities.
- output_router_logits (`bool`, *optional*, defaults to `False`):
- Whether or not the router logits should be returned by the model. Enabling this will also
- allow the model to output the auxiliary loss, including load balancing loss and router z-loss.
- router_aux_loss_coef (`float`, *optional*, defaults to 0.001):
- The aux loss factor for the total loss.
- mlp_only_layers (`list[int]`, *optional*, defaults to `[]`):
- Indicate which layers use Qwen3NextMLP rather than Qwen3NextSparseMoeBlock
- The list contains layer index, from 0 to num_layers-1 if we have num_layers layers
- If `mlp_only_layers` is empty, `decoder_sparse_step` is used to determine the sparsity.
- layer_types (`list[str]`, *optional*):
- Types of each layer (attention or linear).
- enable_qk_norm (`bool`, *optional*, defaults to `False`):
- Whether to apply L2 normalization to the query and key embeddings.
- router_bias (`bool`, *optional*, defaults to `False`):
- Whether to use a bias in the router logits.
- moe_router_score_function (`str`, *optional*, defaults to `"softmax"`):
- The score function used in the MoE router.
- ffn_token_shift (`str`, *optional*): Token shift before FFN/MoE. `None`, `"cat"`, or `"conv"`.
- ffn_intermediate_token_shift (`str`, *optional*): Token shift in MLP before down_proj. `None`, `"cat"`, or `"conv"`.
- attn_token_shift (`str`, *optional*): Token shift before attention. `None`, `"cat"`, or `"conv"`.
- attn_q_token_shift (`str`, *optional*): Token shift on query after projection. `None`, `"cat"`, or `"conv"`.
- attn_k_token_shift (`str`, *optional*): Token shift on key after projection. `None`, `"cat"`, or `"conv"`.
- attn_v_token_shift (`str`, *optional*): Token shift on value after projection. `None`, `"cat"`, or `"conv"`.
- token_shift_conv_size (`int`, *optional*, defaults to 4): Kernel size for token-shift Conv1d.
- token_shift_conv_init (`str`, *optional*, defaults to `"default"`): Init for token-shift Conv1d; `"identity"` for causal identity.
- ```python
- >>> from transformers import Qwen3NextModel, Qwen3NextConfig
-
- >>> # Initializing a Qwen3Next style configuration
- >>> configuration = Qwen3NextConfig()
-
- >>> # Initializing a model from the Qwen3-Next-80B-A3B style configuration
- >>> model = Qwen3NextModel(configuration)
-
- >>> # Accessing the model configuration
- >>> configuration = model.config
- ```
- """
-
- model_type = "qwen3_next"
- keys_to_ignore_at_inference = ["past_key_values"]
-
- base_model_tp_plan = {
- "layers.*.self_attn.q_proj": "colwise",
- "layers.*.self_attn.k_proj": "colwise",
- "layers.*.self_attn.v_proj": "colwise",
- "layers.*.self_attn.o_proj": "rowwise",
- "layers.*.mlp.experts.*.gate_proj": "colwise",
- "layers.*.mlp.experts.*.up_proj": "colwise",
- "layers.*.mlp.experts.*.down_proj": "rowwise",
- "layers.*.mlp.shared_experts.gate_proj": "colwise",
- "layers.*.mlp.shared_experts.up_proj": "colwise",
- "layers.*.mlp.shared_experts.down_proj": "rowwise",
- "layers.*.mlp.gate_proj": "colwise",
- "layers.*.mlp.up_proj": "colwise",
- "layers.*.mlp.down_proj": "rowwise",
- }
- base_model_pp_plan = {
- "embed_tokens": (["input_ids"], ["inputs_embeds"]),
- "layers": (["hidden_states", "attention_mask"], ["hidden_states"]),
- "norm": (["hidden_states"], ["hidden_states"]),
- }
-
- def __init__(
- self,
- vocab_size=151936,
- hidden_size=2048,
- intermediate_size=5632,
- num_hidden_layers=48,
- num_attention_heads=16,
- num_key_value_heads=2,
- hidden_act="silu",
- max_position_embeddings=32768,
- initializer_range=0.02,
- rms_norm_eps=1e-6,
- use_cache=True,
- tie_word_embeddings=False,
- rope_theta=10000.0,
- rope_scaling=None,
- partial_rotary_factor=0.25,
- attention_bias=False,
- attention_dropout=0.0,
- head_dim=256,
- linear_conv_kernel_dim=4,
- linear_key_head_dim=128,
- linear_value_head_dim=128,
- linear_num_key_heads=16,
- linear_num_value_heads=32,
- decoder_sparse_step=1,
- moe_intermediate_size=512,
- shared_expert_intermediate_size=512,
- num_experts_per_tok=10,
- num_experts=512,
- norm_topk_prob=True,
- output_router_logits=False,
- router_aux_loss_coef=0.001,
- mlp_only_layers=[],
- layer_types=None,
- enable_qk_norm=False, # @o2iginal
- router_bias=False, # @o2iginal
- attn_output_gate=False, # @o2iginal
- moe_router_score_function="softmax", # @xcx
- # Cannon layer / token shifting (align with Megatron) @o2iginal
- ffn_token_shift=None,
- ffn_intermediate_token_shift=None,
- attn_token_shift=None,
- attn_q_token_shift=None,
- attn_k_token_shift=None,
- attn_v_token_shift=None,
- token_shift_conv_size=4,
- token_shift_conv_init="default",
- # Separate RoPE for attention vs linear/RNN (GDN): "rope" or "nope"
- attn_position_embedding_type="rope",
- rnn_position_embedding_type="nope",
- # Optional logits scaling for length extrapolation (attention only): None, float, or "log" / "log "
- attn_logits_scaling=None,
- **kwargs,
- ):
- super().__init__(tie_word_embeddings=tie_word_embeddings, **kwargs)
- assert attn_position_embedding_type in ("rope", "nope"), (
- f"attn_position_embedding_type must be 'rope' or 'nope', got {attn_position_embedding_type}"
- )
- assert rnn_position_embedding_type in ("rope", "nope"), (
- f"rnn_position_embedding_type must be 'rope' or 'nope', got {rnn_position_embedding_type}"
- )
- self.attn_position_embedding_type = attn_position_embedding_type
- self.rnn_position_embedding_type = rnn_position_embedding_type
- self.attn_logits_scaling = attn_logits_scaling
- self.vocab_size = vocab_size
- self.max_position_embeddings = max_position_embeddings
- self.hidden_size = hidden_size
- self.intermediate_size = intermediate_size
- self.num_hidden_layers = num_hidden_layers
- self.num_attention_heads = num_attention_heads
- self.num_key_value_heads = num_key_value_heads
- self.hidden_act = hidden_act
- self.initializer_range = initializer_range
- self.rms_norm_eps = rms_norm_eps
- self.use_cache = use_cache
- self.rope_theta = rope_theta
- self.rope_scaling = rope_scaling
- self.partial_rotary_factor = partial_rotary_factor
- self.attention_bias = attention_bias
- self.attention_dropout = attention_dropout
- self.head_dim = head_dim
- rope_config_validation(self)
-
- self.layer_types = layer_types
- if self.layer_types is None:
- interval_pattern = kwargs.get("full_attention_interval", 4)
- self.layer_types = [
- "linear_attention" if bool((i + 1) % interval_pattern) else "full_attention"
- for i in range(self.num_hidden_layers)
- ]
- layer_type_validation(self.layer_types)
-
- # linear attention part
- self.linear_conv_kernel_dim = linear_conv_kernel_dim
- self.linear_key_head_dim = linear_key_head_dim
- self.linear_value_head_dim = linear_value_head_dim
- self.linear_num_key_heads = linear_num_key_heads
- self.linear_num_value_heads = linear_num_value_heads
-
- # MoE arguments
- self.decoder_sparse_step = decoder_sparse_step
- self.moe_intermediate_size = moe_intermediate_size
- self.shared_expert_intermediate_size = shared_expert_intermediate_size
- self.num_experts_per_tok = num_experts_per_tok
- self.num_experts = num_experts
- self.norm_topk_prob = norm_topk_prob
- self.output_router_logits = output_router_logits
- self.router_aux_loss_coef = router_aux_loss_coef
- self.mlp_only_layers = mlp_only_layers
- self.enable_qk_norm = enable_qk_norm
- self.router_bias = router_bias
- self.attn_output_gate = attn_output_gate
- self.moe_router_score_function = moe_router_score_function
- self.enable_qk_norm = enable_qk_norm
- self.router_bias = router_bias
- self.attn_output_gate = attn_output_gate
- self.moe_router_score_function = moe_router_score_function
-
- # Token shifting (cannon layer): None | "cat" | "conv"
- self.ffn_token_shift = ffn_token_shift
- self.ffn_intermediate_token_shift = ffn_intermediate_token_shift
- self.attn_token_shift = attn_token_shift
- self.attn_q_token_shift = attn_q_token_shift
- self.attn_k_token_shift = attn_k_token_shift
- self.attn_v_token_shift = attn_v_token_shift
- self.token_shift_conv_size = token_shift_conv_size
- self.token_shift_conv_init = token_shift_conv_init
-
-__all__ = ["Qwen3NextConfig"]
diff --git a/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/hf2mcore.log b/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/hf2mcore.log
deleted file mode 100644
index ade7dec2973025c61bf98e7d153e30cd368c88e1..0000000000000000000000000000000000000000
--- a/grid_search_0318/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/hf2mcore.log
+++ /dev/null
@@ -1,1277 +0,0 @@
-torchrun --nproc_per_node 1 --nnodes 1 --node_rank 0 --master_addr localhost --master_port 42639 /mnt/ssd/lvzhihao/PostTrain/YuLan-Pretrain/scripts/distributed_checkpoints_convertor/impl/convert.py --tokenizer-type HuggingFaceTokenizer --tokenizer-model /mnt/ssd/cache_tmp/tmp/tmp.9kVbw0fz8t --hf-dir /mnt/ssd/cache_tmp/tmp/tmp.9kVbw0fz8t --mcore2hf --use-gpu --bf16 --normalization RMSNorm --swiglu --disable-bias-linear --seq-length 1 --max-position-embeddings 490000 --attention-backend auto --position-embedding-type rope --kv-channels 64 --group-query-attention --add-qkv-bias --num-layers 56 --hidden-size 1920 --ffn-hidden-size 4800 --num-attention-heads 30 --untie-embeddings-and-output-weights --rotary-base 490000 --rotary-percent 1.00 --num-query-groups 6 --normalization RMSNorm --norm-epsilon 1e-6 --linear-attention-type gated_delta_net --linear-attention-freq [1,1,1,1,1,1,1,1,1,1,1,1,0,1,1,1,1,1,1,1,0,0,1,0,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,0,1,0,0,1,1,1,1,1,1] --linear-conv-kernel-dim 4 --linear-key-head-dim 64 --linear-value-head-dim 64 --linear-num-key-heads 8 --linear-num-value-heads 32 --micro-batch-size 1 --global-batch-size 1024 --train-iters 500000 --weight-decay 0.1 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --clip-grad 1.0 --lr 2.0e-5 --lr-decay-style cosine --min-lr 6.0e-6 --lr-warmup-fraction .001 --lr-decay-iters 430000 --bf16 --tensor-model-parallel-size 1 --pipeline-model-parallel-size 1 --expert-tensor-parallel-size 1 --expert-model-parallel-size 1 --log-interval 100 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --model-type GPT --load-dir /mnt/hdd/lvzhihao/mcore_models/grid_search_0318/checkpoint/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50 --save-dir /mnt/hdd/lvzhihao/mcore_models/grid_search_0318/checkpoint/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/iter_953-hf --dist-ckpt-optim-fully-reshardable --skip-train --use-cpu-initialization --padded-vocab-size 99000 --no-load-optim --no-load-rng --logging-level 1 --attention-backend auto --synchronizer mcore_gdn_moe --pretrain-script mcore_gdn_moe.model_provider --debug --max-shard-size 20GB
-W0320 09:19:34.443000 3250 .venv/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2425] TORCH_CUDA_ARCH_LIST is not set, all archs for visible cards are included for compilation.
-W0320 09:19:34.443000 3250 .venv/lib/python3.10/site-packages/torch/utils/cpp_extension.py:2425] If this is not desired, please set os.environ['TORCH_CUDA_ARCH_LIST'] to specific architectures.
-fused_indices_to_multihot has reached end of life. Please migrate to a non-experimental function.
-Current Python version 3.10 is below the recommended 3.11 version. It is recommended to upgrade to Python 3.11 or higher for the best experience.
-Warning: Pai-Megatron-Patch arguments not available, some arguments may not be recognized
-using world size: 1, data-parallel size: 1, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1
-Number of virtual stages per pipeline stage: None
-accumulate and all-reduce gradients in fp32 for bfloat16 data type.
-using torch.bfloat16 for parameters ...
------------------------- arguments ------------------------
- account_for_embedding_in_pipeline_split ......... False
- account_for_loss_in_pipeline_split .............. False
- accumulate_allreduce_grads_in_fp32 .............. True
- activation_func_clamp_value ..................... None
- adam_beta1 ...................................... 0.9
- adam_beta2 ...................................... 0.95
- adam_eps ........................................ 1e-08
- adamw_lr_mup_scaler ............................. False
- add_bias_linear ................................. False
- add_position_embedding .......................... True
- add_qkv_bias .................................... True
- adlr_autoresume ................................. False
- adlr_autoresume_interval ........................ 1000
- align_grad_reduce ............................... True
- align_param_gather .............................. False
- allow_ambiguous_pad_tokens ...................... False
- app_tag_run_name ................................ None
- app_tag_run_version ............................. 0.0.0
- apply_layernorm_1p .............................. False
- apply_query_key_layer_scaling ................... False
- apply_residual_connection_post_layernorm ........ False
- apply_rope_fusion ............................... True
- async_save ...................................... None
- async_tensor_model_parallel_allreduce ........... True
- attention_backend ............................... AttnBackend.auto
- attention_dropout ............................... 0.1
- attention_output_gate ........................... False
- attention_softmax_in_fp32 ....................... False
- attn_k_token_shift .............................. None
- attn_output_gate ................................ None
- attn_output_gate_rand_init ...................... False
- attn_q_token_shift .............................. None
- attn_token_shift ................................ None
- attn_v_token_shift .............................. None
- auto_detect_ckpt_format ......................... False
- auto_generate_cu_seqlens ........................ False
- auto_model ...................................... AutoModelForCausalLM
- barrier_with_L1_time ............................ True
- benchmark_eval .................................. False
- benchmark_global_batch .......................... None
- benchmark_interval .............................. None
- benchmark_micro_batch ........................... None
- benchmark_sequence_length ....................... None
- benchmark_tasks ................................. None
- bert_binary_head ................................ True
- bert_embedder_type .............................. megatron
- bert_load ....................................... None
- bf16 ............................................ True
- bias_dropout_fusion ............................. True
- bias_gelu_fusion ................................ False
- bias_swiglu_fusion .............................. True
- biencoder_projection_dim ........................ 0
- biencoder_shared_query_context_model ............ False
- block_data_path ................................. None
- cache_mla_latents ............................... False
- calc_ft_timeouts ................................ False
- calculate_per_token_loss ........................ False
- check_for_large_grads ........................... False
- check_for_nan_in_loss_and_grad .................. True
- check_for_spiky_loss ............................ False
- check_weight_hash_across_dp_replicas_interval ... None
- ckpt_assume_constant_structure .................. False
- ckpt_convert_format ............................. None
- ckpt_convert_save ............................... None
- ckpt_convert_update_legacy_dist_opt_format ...... False
- ckpt_format ..................................... torch_dist
- ckpt_fully_parallel_load ........................ False
- ckpt_fully_parallel_save ........................ True
- ckpt_fully_parallel_save_deprecated ............. False
- ckpt_step ....................................... None
- classes_fraction ................................ 1.0
- clip_grad ....................................... 1.0
- clone_scatter_output_in_embedding ............... True
- config_logger_dir ...............................
- consumed_train_samples .......................... 0
- consumed_valid_samples .......................... 0
- context_parallel_size ........................... 1
- cp_comm_type .................................... ['p2p']
- create_attention_mask_in_dataloader ............. True
- cross_entropy_fusion_impl ....................... native
- cross_entropy_loss_fusion ....................... False
- cuda_graph_impl ................................. none
- cuda_graph_scope ................................ []
- cuda_graph_warmup_steps ......................... 3
- data_args_path .................................. None
- data_cache_path ................................. None
- data_parallel_random_init ....................... False
- data_parallel_sharding_strategy ................. no_shard
- data_parallel_size .............................. 1
- data_path ....................................... None
- data_per_class_fraction ......................... 1.0
- data_sharding ................................... True
- dataloader_type ................................. single
- ddp_average_in_collective ....................... False
- ddp_bucket_size ................................. None
- ddp_num_buckets ................................. None
- ddp_pad_buckets_for_high_nccl_busbw ............. False
- debug ........................................... True
- decode_only_cuda_graphs ......................... False
- decoder_first_pipeline_num_layers ............... None
- decoder_last_pipeline_num_layers ................ None
- decoder_num_layers .............................. None
- decoder_seq_length .............................. None
- decoupled_lr .................................... None
- decoupled_min_lr ................................ None
- decrease_batch_size_if_needed ................... False
- defer_embedding_wgrad_compute ................... False
- delay_wgrad_compute ............................. False
- deprecated_use_mcore_models ..................... False
- deterministic_mode .............................. False
- dino_bottleneck_size ............................ 256
- dino_freeze_last_layer .......................... 1
- dino_head_hidden_size ........................... 2048
- dino_local_crops_number ......................... 10
- dino_local_img_size ............................. 96
- dino_norm_last_layer ............................ False
- dino_teacher_temp ............................... 0.07
- dino_warmup_teacher_temp ........................ 0.04
- dino_warmup_teacher_temp_epochs ................. 30
- disable_attn_output_gate ........................ False
- disable_bf16_reduced_precision_matmul ........... False
- disable_chunked_prefill ......................... False
- disable_explicit_attention_mask ................. False
- disable_mamba_mem_eff_path ...................... False
- disable_straggler_on_startup .................... False
- disable_symmetric_registration .................. False
- dist_ckpt_format_deprecated ..................... None
- dist_ckpt_optim_fully_reshardable ............... True
- dist_ckpt_save_pre_mcore_014 .................... False
- dist_ckpt_strictness ............................ assume_ok_unexpected
- distrib_optim_fully_reshardable_mem_efficient ... False
- distribute_saved_activations .................... False
- distributed_backend ............................. nccl
- distributed_timeout_minutes ..................... 10
- distributed_timeout_seconds_after_init .......... None
- document_packing_algorithm ...................... random
- dryrun .......................................... False
- dump_param_to_param_group_map ................... None
- emb_deviation_loss_coeff ........................ 0
- emb_deviation_type .............................. None
- embedding_init_method_std ....................... None
- embedding_path .................................. None
- empty_unused_memory_level ....................... 0
- enable_cuda_graph ............................... False
- enable_debug_logging ............................ False
- enable_experimental ............................. False
- enable_ft_package ............................... False
- enable_full_sharding_in_hsdp .................... False
- enable_gloo_process_groups ...................... True
- enable_msc ...................................... True
- enable_one_logger ............................... True
- encoder_num_layers .............................. 56
- encoder_seq_length .............................. 1
- end_weight_decay ................................ 0.1
- eod_mask_loss ................................... False
- error_injection_rate ............................ 0
- error_injection_type ............................ transient_error
- eval_interval ................................... 1000
- eval_iters ...................................... 10
- evidence_data_path .............................. None
- exit_duration_in_mins ........................... None
- exit_interval ................................... None
- exit_on_missing_checkpoint ...................... False
- exit_signal_handler ............................. False
- exp_avg_dtype ................................... torch.float32
- exp_avg_sq_dtype ................................ torch.float32
- expert_model_parallel_size ...................... 1
- expert_tensor_parallel_size ..................... 1
- external_cuda_graph ............................. False
- ffn_hidden_size ................................. 4800
- ffn_intermediate_token_shift .................... None
- ffn_token_shift ................................. None
- fine_grained_activation_offloading .............. False
- finetune ........................................ False
- first_last_layers_bf16 .......................... False
- flash_decode .................................... False
- fp16 ............................................ False
- fp16_lm_cross_entropy ........................... False
- fp32_residual_connection ........................ False
- fp4 ............................................. None
- fp4_param ....................................... False
- fp4_recipe ...................................... nvfp4
- fp8 ............................................. None
- fp8_amax_compute_algo ........................... most_recent
- fp8_amax_history_len ............................ 1
- fp8_interval .................................... 1
- fp8_margin ...................................... 0
- fp8_param_gather ................................ False
- fp8_recipe ...................................... delayed
- fp8_wgrad ....................................... True
- freeze_layernorm_weight ......................... False
- freeze_non_mamba ................................ False
- fsdp_double_buffer .............................. False
- full_validation ................................. False
- gdn_cp_impl ..................................... cp2hp
- geglu ........................................... False
- global_batch_size ............................... 1024
- glu_linear_offset ............................... 0.0
- grad_reduce_in_bf16 ............................. False
- gradient_accumulation_fusion .................... True
- gradient_reduce_div_fusion ...................... True
- group_query_attention ........................... True
- grpo_clamp_eps_lower ............................ 0.01
- grpo_clamp_eps_upper ............................ 0.01
- grpo_default_temperature ........................ 1.0
- grpo_default_top_p .............................. 0
- grpo_entropy_term_weight ........................ 0.0
- grpo_filter_groups_with_same_reward ............. False
- grpo_group_size ................................. 2
- grpo_iterations ................................. 2
- grpo_kl_beta .................................... 0.001
- grpo_prompts_per_step ........................... 32
- head_lr_mult .................................... 1.0
- heterogeneous_layers_config_encoded_json ........ None
- heterogeneous_layers_config_path ................ None
- hf_dir .......................................... /mnt/ssd/cache_tmp/tmp/tmp.9kVbw0fz8t
- hidden_dropout .................................. 0.1
- hidden_size ..................................... 1920
- hierarchical_context_parallel_sizes ............. None
- high_priority_stream_groups ..................... []
- hybrid_attention_ratio .......................... 0.0
- hybrid_context_parallel ......................... False
- hybrid_mlp_ratio ................................ 0.0
- hybrid_override_pattern ......................... None
- hysteresis ...................................... 2
- ict_head_size ................................... None
- ict_load ........................................ None
- img_h ........................................... 224
- img_w ........................................... 224
- increase_log_level_interval ..................... 1000
- increase_log_level_iters ........................ 5
- indexer_batch_size .............................. 128
- indexer_log_interval ............................ 1000
- inference_batch_times_seqlen_threshold .......... -1
- inference_dynamic_batching ...................... False
- inference_dynamic_batching_block_size ........... 256
- inference_dynamic_batching_buffer_guaranteed_fraction 0.2
- inference_dynamic_batching_buffer_overflow_factor None
- inference_dynamic_batching_buffer_size_gb ....... 40.0
- inference_dynamic_batching_max_requests_override None
- inference_dynamic_batching_max_tokens_override .. None
- inference_dynamic_batching_num_cuda_graphs ...... 16
- inference_dynamic_batching_track_paused_request_events False
- inference_dynamic_batching_unified_memory_level . 0
- inference_max_batch_size ........................ 8
- inference_max_seq_length ........................ 2560
- inference_rng_tracker ........................... False
- init_method_std ................................. 0.006
- init_method_xavier_uniform ...................... False
- init_model_with_meta_device ..................... False
- initial_loss_scale .............................. 4294967296
- inprocess_active_world_size ..................... 1
- inprocess_barrier_timeout ....................... 120
- inprocess_completion_timeout .................... 120
- inprocess_empty_cuda_cache ...................... False
- inprocess_granularity ........................... node
- inprocess_hard_timeout .......................... 90
- inprocess_heartbeat_interval .................... 30
- inprocess_heartbeat_timeout ..................... 60
- inprocess_last_call_wait ........................ 1
- inprocess_max_iterations ........................ None
- inprocess_monitor_process_interval .............. 1.0
- inprocess_monitor_thread_interval ............... 1.0
- inprocess_progress_watchdog_interval ............ 1.0
- inprocess_restart ............................... False
- inprocess_soft_timeout .......................... 60
- inprocess_termination_grace_time ................ 1
- is_hybrid_model ................................. False
- iter_per_epoch .................................. 1250
- iterations_to_skip .............................. []
- keep_fp8_transpose_cache ........................ False
- kitchen_config_file ............................. None
- kitchen_recipe_number ........................... None
- kv_channels ..................................... 64
- kv_lora_rank .................................... 32
- langrl_env_config ............................... None
- langrl_external_server .......................... False
- langrl_inference_server_conversation_template ... None
- langrl_inference_server_type .................... inplace_megatron
- lazy_mpu_init ................................... None
- legacy_tokenizer ................................ False
- linear_attention_freq ........................... [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1]
- linear_attention_type ........................... gated_delta_net
- linear_conv_kernel_dim .......................... 4
- linear_key_head_dim ............................. 64
- linear_num_key_heads ............................ 8
- linear_num_value_heads .......................... 32
- linear_value_head_dim ........................... 64
- load ............................................ None
- load_complemental_dataset ....................... None
- load_dir ........................................ /mnt/hdd/lvzhihao/mcore_models/grid_search_0318/checkpoint/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50
- load_main_params_from_ckpt ...................... None
- local_rank ...................................... 0
- log_energy ...................................... False
- log_hidden_states ............................... []
- log_interval .................................... 100
- log_loss_scale_to_tensorboard ................... True
- log_memory_to_tensorboard ....................... False
- log_num_zeros_in_grad ........................... False
- log_params ...................................... []
- log_params_norm ................................. False
- log_per_module_grad_rms ......................... False
- log_per_module_update_rms ....................... False
- log_progress .................................... False
- log_straggler ................................... False
- log_throughput .................................. False
- log_timers_to_tensorboard ....................... False
- log_validation_ppl_to_tensorboard ............... False
- log_world_size_to_tensorboard ................... False
- logging_level ................................... 1
- loss_scale ...................................... None
- loss_scale_window ............................... 1000
- lr .............................................. 2e-05
- lr_decay_iters .................................. 430000
- lr_decay_samples ................................ None
- lr_decay_style .................................. cosine
- lr_warmup_fraction .............................. 0.001
- lr_warmup_init .................................. 0.0
- lr_warmup_iters ................................. 0
- lr_warmup_samples ............................... 0
- lr_wsd_decay_iters .............................. None
- lr_wsd_decay_samples ............................ None
- lr_wsd_decay_style .............................. exponential
- main_grads_dtype ................................ torch.float32
- main_params_dtype ............................... torch.float32
- make_vocab_size_divisible_by .................... 128
- mamba_disable_cp ................................ False
- mamba_expand .................................... 2
- mamba_head_dim .................................. 64
- mamba_num_groups ................................ 8
- mamba_num_heads ................................. None
- mamba_state_dim ................................. 128
- manual_gc ....................................... False
- manual_gc_eval .................................. True
- manual_gc_interval .............................. 0
- mask_factor ..................................... 1.0
- mask_prob ....................................... 0.15
- mask_type ....................................... random
- masked_softmax_fusion ........................... True
- max_position_embeddings ......................... 490000
- max_seqlen_per_cp_rank .......................... None
- max_shard_size .................................. 20GB
- max_tokens_to_oom ............................... 12000
- mcore2hf ........................................ True
- memory_snapshot_path ............................ None
- merge_file ...................................... None
- micro_batch_size ................................ 1
- microbatch_group_size_per_vp_stage .............. None
- mid_level_dataset_surplus ....................... 0.005
- min_loss_scale .................................. 1.0
- min_lr .......................................... 6e-06
- min_offloaded_tensor_size ....................... 1048576
- mlp_chunks_for_prefill .......................... 1
- mmap_bin_files .................................. True
- mock_data ....................................... False
- model_type ...................................... GPT
- moe_apply_probs_on_input ........................ False
- moe_aux_loss_coeff .............................. 0.0
- moe_deepep_num_sms .............................. 20
- moe_enable_deepep ............................... False
- moe_expert_capacity_factor ...................... None
- moe_extended_tp ................................. False
- moe_ffn_hidden_size ............................. None
- moe_flex_dispatcher_backend ..................... deepep
- moe_grouped_gemm ................................ False
- moe_hybridep_num_sms ............................ 16
- moe_input_jitter_eps ............................ None
- moe_layer_freq .................................. 1
- moe_layer_recompute ............................. False
- moe_pad_expert_input_to_capacity ................ False
- moe_pad_experts_for_cuda_graph_inference ........ False
- moe_per_layer_logging ........................... False
- moe_permute_fusion .............................. False
- moe_router_bias_update_method ................... sign
- moe_router_bias_update_rate ..................... 0.001
- moe_router_dtype ................................ None
- moe_router_enable_expert_bias ................... False
- moe_router_force_load_balancing ................. False
- moe_router_fusion ............................... False
- moe_router_group_topk ........................... None
- moe_router_load_balancing_type .................. aux_loss
- moe_router_num_groups ........................... None
- moe_router_padding_for_fp8 ...................... False
- moe_router_padding_for_quantization ............. False
- moe_router_pre_softmax .......................... False
- moe_router_score_function ....................... softmax
- moe_router_topk ................................. 2
- moe_router_topk_scaling_factor .................. None
- moe_shared_expert_gate .......................... False
- moe_shared_expert_intermediate_size ............. None
- moe_shared_expert_overlap ....................... False
- moe_token_dispatcher_type ....................... allgather
- moe_token_drop_policy ........................... probs
- moe_upcycling_granularity ....................... 1
- moe_use_legacy_grouped_gemm ..................... False
- moe_use_upcycling ............................... False
- moe_z_loss_coeff ................................ None
- mrope_section ................................... None
- mscale .......................................... 1.0
- mscale_all_dim .................................. 0.0
- mtp_linear_attention_type ....................... None
- mtp_loss_scaling_factor ......................... 0.1
- mtp_num_layers .................................. None
- multi_latent_attention .......................... False
- multiple_validation_sets ........................ False
- muon_ball_momentum .............................. 0.9
- muon_ball_msign_steps ........................... 5
- muon_ball_power_iteration_steps ................. 10
- muon_ball_qkv_split_mode ........................ component
- muon_ball_radius_mode ........................... spectral_mup
- muon_ball_retract_alpha ......................... 0.05
- muon_ball_retract_mode .......................... hard
- muon_ball_scale_mode ............................ spectral_mup
- muon_ball_split_fc1 ............................. True
- muon_ball_split_moe_experts ..................... True
- muon_ball_split_qkv ............................. True
- muon_ball_use_nesterov .......................... True
- muon_extra_scale_factor ......................... 1.0
- muon_fp32_matmul_prec ........................... medium
- muon_momentum ................................... 0.9
- muon_num_ns_steps ............................... 5
- muon_qkv_split_mode ............................. component
- muon_scale_mode ................................. spectral_mup
- muon_scale_vectorized_mode ...................... full
- muon_split_fc1 .................................. True
- muon_split_moe_experts .......................... True
- muon_split_qkv .................................. True
- muon_tp_mode .................................... blockwise
- muon_use_nesterov ............................... False
- muon_vectorize .................................. []
- muon_vectorize_attn_dim ......................... hidden_size
- nccl_all_reduce_for_prefill ..................... False
- nccl_communicator_config_path ................... None
- nccl_ub ......................................... False
- no_load_optim ................................... True
- no_load_rng ..................................... True
- no_load_scheduler ............................... None
- no_persist_layer_norm ........................... False
- no_rope_freq .................................... None
- no_save_optim ................................... None
- no_save_rng ..................................... None
- no_save_step_one ................................ None
- no_weight_decay_cond_type ....................... None
- non_persistent_ckpt_type ........................ None
- non_persistent_global_ckpt_dir .................. None
- non_persistent_local_ckpt_algo .................. fully_parallel
- non_persistent_local_ckpt_dir ................... None
- non_persistent_save_interval .................... None
- norm_epsilon .................................... 1e-06
- normalization ................................... RMSNorm
- num_attention_heads ............................. 30
- num_channels .................................... 3
- num_classes ..................................... 1000
- num_dataset_builder_threads ..................... 1
- num_distributed_optimizer_instances ............. 1
- num_experts ..................................... None
- num_hf_saver .................................... None
- num_layers ...................................... 56
- num_layers_at_end_in_bf16 ....................... 1
- num_layers_at_start_in_bf16 ..................... 1
- num_layers_per_virtual_pipeline_stage ........... None
- num_query_groups ................................ 6
- num_virtual_stages_per_pipeline_rank ............ None
- num_workers ..................................... 2
- object_storage_cache_path ....................... None
- offload_modules ................................. []
- one_logger_async ................................ False
- one_logger_project .............................. megatron-lm
- one_logger_run_name ............................. None
- onnx_safe ....................................... None
- openai_gelu ..................................... False
- optimizer ....................................... adam
- optimizer_cpu_offload ........................... False
- optimizer_offload_fraction ...................... 1.0
- output_bert_embeddings .......................... False
- overlap_cpu_optimizer_d2h_h2d ................... False
- overlap_grad_reduce ............................. False
- overlap_moe_expert_parallel_comm ................ False
- overlap_p2p_comm ................................ False
- overlap_p2p_comm_warmup_flush ................... False
- overlap_param_gather ............................ False
- overlap_param_gather_with_optimizer_step ........ False
- override_hf_eod_token_id ........................ None
- override_opt_param_scheduler .................... False
- padded_vocab_size ............................... 99000
- params_dtype .................................... torch.bfloat16
- patch_dim ....................................... 16
- per_split_data_args_path ........................ None
- perform_initialization .......................... True
- perform_rl_step ................................. False
- pin_cpu_grads ................................... True
- pin_cpu_params .................................. True
- pipeline_model_parallel_comm_backend ............ None
- pipeline_model_parallel_layout .................. None
- pipeline_model_parallel_size .................... 1
- position_embedding_type ......................... rope
- pretrain_script ................................. mcore_gdn_moe.model_provider
- pretrained_checkpoint ........................... None
- profile ......................................... False
- profile_ranks ................................... [0]
- profile_step_end ................................ 12
- profile_step_start .............................. 10
- q_lora_rank ..................................... None
- qk_head_dim ..................................... 128
- qk_l2_norm ...................................... False
- qk_layernorm .................................... False
- qk_pos_emb_head_dim ............................. 64
- query_in_block_prob ............................. 0.1
- quick_geglu ..................................... False
- rampup_batch_size ............................... None
- rank ............................................ 0
- recompute_granularity ........................... None
- recompute_method ................................ None
- recompute_modules ............................... None
- recompute_num_layers ............................ None
- record_memory_history ........................... False
- relative_attention_max_distance ................. 128
- relative_attention_num_buckets .................. 32
- reparam_checkpoint .............................. None
- reparam_fallback_value .......................... None
- reparam_keys .................................... None
- replication ..................................... False
- replication_factor .............................. 2
- replication_jump ................................ None
- rerun_mode ...................................... validate_results
- reset_attention_mask ............................ False
- reset_iteration_one_to_zero ..................... False
- reset_position_ids .............................. False
- result_rejected_tracker_filename ................ None
- retriever_report_topk_accuracies ................ []
- retriever_score_scaling ......................... False
- retriever_seq_length ............................ 256
- retro_add_retriever ............................. False
- retro_attention_gate ............................ 1
- retro_cyclic_train_iters ........................ None
- retro_encoder_attention_dropout ................. 0.1
- retro_encoder_hidden_dropout .................... 0.1
- retro_encoder_layers ............................ 2
- retro_num_neighbors ............................. 2
- retro_num_retrieved_chunks ...................... 2
- retro_project_dir ............................... None
- retro_verify_neighbor_count ..................... True
- reuse_grad_buf_for_mxfp8_param_ag ............... False
- rl_calculate_intra_group_similarity ............. False
- rl_importance_sampling_truncation_coef .......... None
- rl_inference_logprobs_is_correction ............. False
- rl_offload_kv_cache_during_training ............. False
- rl_offload_optimizer_during_inference ........... False
- rl_partial_rollouts ............................. False
- rl_prompts_per_eval ............................. 32
- rl_remove_kv_cache_during_training .............. False
- rl_reset_cuda_graphs ............................ False
- rl_sequence_packing_algo ........................ fifo
- rl_sequence_packing_bin_size .................... 8192
- rl_use_sequence_packing ......................... False
- rope_scaling_factor ............................. 8.0
- rope_type ....................................... None
- rotary_base ..................................... 490000
- rotary_interleaved .............................. False
- rotary_percent .................................. 1.0
- rotary_scaling_factor ........................... 1.0
- rotary_seq_len_interpolation_factor ............. None
- run_workload_inspector_server ................... False
- sample_rate ..................................... 1.0
- save ............................................ None
- save_after_load ................................. False
- save_dir ........................................ /mnt/hdd/lvzhihao/mcore_models/grid_search_0318/checkpoint/yulan-gdn-sft-1b-sl65536-lr5e-6-gbs16-mb1-tp2-pp1-cp4-fresh-2026.03.19-18.56.50/iter_953-hf
- save_interval ................................... 10000
- save_retain_interval ............................ None
- scatter_gather_tensors_in_pipeline .............. True
- seed ............................................ 1234
- seq_length ...................................... 1
- sequence_parallel ............................... False
- sft ............................................. False
- sft_tokenizer_prompt_format ..................... nemotron-h-aligned
- sgd_momentum .................................... 0.9
- sharp_enabled_group ............................. None
- short_seq_prob .................................. 0.1
- skip_train ...................................... True
- skipped_train_samples ........................... 0
- softmax_type .................................... vanilla
- spec ............................................ None
- spectral_ball_momentum .......................... 0.9
- spectral_ball_msign_steps ....................... 8
- spectral_ball_power_iteration_steps ............. 20
- spectral_ball_qkv_split_mode .................... component
- spectral_ball_radius_mode ....................... spectral_mup
- spectral_ball_retract_alpha ..................... 0.05
- spectral_ball_retract_mode ...................... hard
- spectral_ball_scale_mode ........................ spectral_mup
- spectral_ball_solver ............................ bisection
- spectral_ball_solver_max_iterations ............. 20
- spectral_ball_solver_tolerance_f ................ 1e-08
- spectral_ball_split_fc1 ......................... True
- spectral_ball_split_moe_experts ................. True
- spectral_ball_split_qkv ......................... True
- spectral_ball_use_nesterov ...................... True
- spectral_mup_init ............................... False
- split ........................................... None
- split_expert_init ............................... True
- split_fc1_init .................................. True
- split_qkv_init .................................. True
- split_qkv_init_mode ............................. group
- sqreglu ......................................... False
- squared_relu .................................... False
- start_samples ................................... None
- start_weight_decay .............................. 0.1
- straggler_ctrlr_port ............................ 65535
- straggler_minmax_count .......................... 1
- strict_fsdp_dtensor_load ........................ True
- suggested_communication_unit_size ............... None
- swanlab_exp_name ................................
- swanlab_project .................................
- swanlab_save_dir ................................
- swanlab_workspace ...............................
- swiglu .......................................... True
- swin_backbone_type .............................. tiny
- symmetric_ar_type ............................... None
- synchronizer .................................... mcore_gdn_moe
- target_ckpt_format .............................. torch_dist
- te_rng_tracker .................................. False
- tensor_model_parallel_size ...................... 1
- tensorboard_dir ................................. None
- tensorboard_log_interval ........................ 1
- tensorboard_queue_size .......................... 1000
- test_data_path .................................. None
- test_mode ....................................... False
- tiktoken_num_special_tokens ..................... 1000
- tiktoken_pattern ................................ None
- tiktoken_special_tokens ......................... None
- timing_log_level ................................ 0
- timing_log_option ............................... minmax
- titles_data_path ................................ None
- token_shift_conv_init ........................... default
- token_shift_conv_size ........................... 4
- tokenizer_metadata .............................. None
- tokenizer_model ................................. /mnt/ssd/cache_tmp/tmp/tmp.9kVbw0fz8t
- tokenizer_type .................................. HuggingFaceTokenizer
- torch_fsdp2_reshard_after_forward ............... True
- tp_comm_bootstrap_backend ....................... nccl
- tp_comm_bulk_dgrad .............................. True
- tp_comm_bulk_wgrad .............................. True
- tp_comm_overlap ................................. False
- tp_comm_overlap_ag .............................. True
- tp_comm_overlap_cfg ............................. None
- tp_comm_overlap_rs .............................. True
- tp_comm_overlap_rs_dgrad ........................ False
- tp_comm_split_ag ................................ True
- tp_comm_split_rs ................................ True
- train_data_path ................................. None
- train_iters ..................................... 500000
- train_samples ................................... None
- train_sync_interval ............................. None
- transformer_impl ................................ transformer_engine
- transformer_pipeline_model_parallel_size ........ 1
- trust_remote_code ............................... False
- untie_embeddings_and_output_weights ............. True
- use_checkpoint_args ............................. False
- use_checkpoint_opt_param_scheduler .............. False
- use_cpu_initialization .......................... True
- use_dist_ckpt ................................... True
- use_dist_ckpt_deprecated ........................ False
- use_distributed_optimizer ....................... False
- use_flash_attn .................................. False
- use_fused_weighted_squared_relu ................. False
- use_gpu ......................................... True
- use_legacy_models ............................... False
- use_megatron_fsdp ............................... False
- use_mp_args_from_checkpoint_args ................ False
- use_one_sent_docs ............................... False
- use_persistent_ckpt_worker ...................... False
- use_precision_aware_optimizer ................... False
- use_pytorch_profiler ............................ False
- use_ring_exchange_p2p ........................... False
- use_rope_scaling ................................ False
- use_rotary_position_embeddings .................. False
- use_sharp ....................................... False
- use_te_activation_func .......................... False
- use_tokenizer_model_from_checkpoint_args ........ True
- use_torch_fsdp2 ................................. False
- use_torch_optimizer_for_cpu_offload ............. False
- use_tp_pp_dp_mapping ............................ False
- v_head_dim ...................................... 128
- valid_data_path ................................. None
- variable_seq_lengths ............................ False
- virtual_pipeline_model_parallel_size ............ None
- vision_backbone_type ............................ vit
- vision_pretraining .............................. False
- vision_pretraining_type ......................... classify
- vocab_extra_ids ................................. 0
- vocab_file ...................................... None
- vocab_size ...................................... None
- wandb_entity ....................................
- wandb_exp_name ..................................
- wandb_project ...................................
- wandb_save_dir ..................................
- weight_decay .................................... 0.1
- weight_decay_incr_style ......................... constant
- wgrad_deferral_limit ............................ 0
- window_attn_skip_freq ........................... None
- window_size ..................................... None
- word_embedding_dropout_prob ..................... 0.0
- world_size ...................................... 1
- yaml_cfg ........................................ None
--------------------- end of arguments ---------------------
-INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1024
-> building HuggingFaceTokenizer tokenizer ...
-You are using the default legacy behaviour of the ": 43,
- "",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- },
- "eos_token": {
- "content": "",
- "lstrip": false,
- "normalized": false,
- "rstrip": false,
- "single_word": false
- },
- "pad_token": {
- "content": "",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- {
- "id": 2,
- "content": "<|endoftext|>",
- "single_word": false,
- "lstrip": false,
- "rstrip": false,
- "normalized": false,
- "special": true
- },
- {
- "id": 102,
- "content": "",
- "type_id": 0
- }
- },
- {
- "Sequence": {
- "id": "A",
- "type_id": 0
- }
- }
- ],
- "pair": [
- {
- "SpecialToken": {
- "id": "",
- "type_id": 0
- }
- },
- {
- "Sequence": {
- "id": "A",
- "type_id": 0
- }
- },
- {
- "SpecialToken": {
- "id": "",
- "type_id": 1
- }
- },
- {
- "Sequence": {
- "id": "B",
- "type_id": 1
- }
- }
- ],
- "special_tokens": {
- "": {
- "id": "",
- "ids": [
- 1
- ],
- "tokens": [
- ""
- ]
- }
- }
- },
- "decoder": {
- "type": "Sequence",
- "decoders": [
- {
- "type": "Replace",
- "pattern": {
- "String": "▁"
- },
- "content": " "
- },
- {
- "type": "ByteFallback"
- },
- {
- "type": "Fuse"
- },
- {
- "type": "Strip",
- "content": " ",
- "start": 1,
- "stop": 0
- }
- ]
- },
- "model": {
- "type": "BPE",
- "dropout": null,
- "unk_token": "": 1,
- "<|endoftext|>": 2,
- "": 7,
- "
": 8,
- "": 9,
- "
": 10,
- "": 11,
- "
": 12,
- "": 13,
- " ": 14,
- "": 15,
- "": 16,
- "": 17,
- "": 18,
- " ": 19,
- "
": 21,
- "": 20,
- "
": 22,
- " ": 23,
- "": 24,
- " ": 31,
- "": 32,
- "
": 33,
- "
": 34,
- "
": 35,
- "": 36,
- "": 37,
- "": 38,
- "
": 39,
- "": 40,
- "": 41,
- "
": 42,
- "