| _target_: nemo.collections.llm.gpt.model.ssm.MambaModel |
| config: |
| _cpu_offloading_context: null |
| _target_: nemo.collections.llm.gpt.model.ssm.NemotronHConfig8B |
| account_for_embedding_in_pipeline_split: false |
| account_for_loss_in_pipeline_split: false |
| activation_func: |
| _call_: false |
| _target_: nemo.collections.llm.gpt.model.ssm.NemotronHConfigBase.<lambda> |
| activation_func_fp8_input_store: false |
| add_bias_linear: false |
| add_qkv_bias: false |
| apply_query_key_layer_scaling: false |
| apply_residual_connection_post_layernorm: false |
| apply_rope_fusion: true |
| async_tensor_model_parallel_allreduce: false |
| attention_backend: |
| _call_: true |
| _target_: megatron.core.transformer.enums.AttnBackend |
| attention_dropout: 0.0 |
| attention_softmax_in_fp32: false |
| autocast_dtype: null |
| barrier_with_L1_time: true |
| batch_p2p_comm: true |
| batch_p2p_sync: true |
| bf16: true |
| bias_activation_fusion: false |
| bias_dropout_fusion: true |
| calculate_per_token_loss: false |
| clone_scatter_output_in_embedding: true |
| config_logger_dir: '' |
| context_parallel_size: 1 |
| cp_comm_type: null |
| cpu_offloading: false |
| cpu_offloading_activations: true |
| cpu_offloading_num_layers: 0 |
| cpu_offloading_weights: true |
| cross_entropy_fusion_impl: native |
| cross_entropy_loss_fusion: true |
| cuda_graph_retain_backward_graph: false |
| cuda_graph_scope: full |
| cuda_graph_use_single_mempool: false |
| cuda_graph_warmup_steps: 3 |
| data_step_fn: |
| _call_: false |
| _target_: nemo.collections.llm.gpt.model.base.gpt_data_step |
| deallocate_pipeline_outputs: true |
| defer_embedding_wgrad_compute: false |
| deterministic_mode: false |
| disable_parameter_transpose_cache: false |
| distribute_saved_activations: null |
| enable_autocast: false |
| enable_cuda_graph: false |
| expert_model_parallel_size: 1 |
| expert_tensor_parallel_size: null |
| external_cuda_graph: false |
| ffn_hidden_size: 21504 |
| finalize_model_grads_func: null |
| first_last_layers_bf16: true |
| flash_decode: false |
| forward_step_fn: |
| _call_: false |
| _target_: nemo.collections.llm.gpt.model.ssm.ssm_forward_step |
| fp16: false |
| fp16_lm_cross_entropy: false |
| fp32_residual_connection: false |
| fp8: null |
| fp8_amax_compute_algo: most_recent |
| fp8_amax_history_len: 1 |
| fp8_dot_product_attention: false |
| fp8_interval: 1 |
| fp8_margin: 0 |
| fp8_multi_head_attention: false |
| fp8_recipe: delayed |
| fp8_wgrad: true |
| gated_linear_unit: false |
| get_attention_mask_from_fusion: false |
| grad_scale_func: null |
| grad_sync_func: null |
| gradient_accumulation_fusion: false |
| hidden_dropout: 0.0 |
| hidden_size: 4096 |
| hierarchical_context_parallel_sizes: null |
| hybrid_attention_ratio: 0.0 |
| hybrid_mlp_ratio: 0.0 |
| hybrid_override_pattern: M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M*-M-M-M-M-M- |
| inference_rng_tracker: false |
| init_method: null |
| init_method_std: 0.02 |
| init_model_with_meta_device: false |
| is_hybrid_model: true |
| kv_channels: null |
| layernorm_epsilon: 1.0e-05 |
| layernorm_zero_centered_gamma: false |
| make_vocab_size_divisible_by: 128 |
| mamba_head_dim: 64 |
| mamba_nheads: 128 |
| mamba_num_groups: 8 |
| mamba_state_dim: 128 |
| mapping_type: nvidia-hybrid-nemotronh |
| masked_softmax_fusion: true |
| memory_efficient_layer_norm: false |
| microbatch_group_size_per_vp_stage: 1 |
| moe_aux_loss_coeff: 0 |
| moe_enable_deepep: false |
| moe_expert_capacity_factor: null |
| moe_extended_tp: false |
| moe_ffn_hidden_size: null |
| moe_grouped_gemm: false |
| moe_input_jitter_eps: null |
| moe_layer_freq: 1 |
| moe_layer_recompute: false |
| moe_pad_expert_input_to_capacity: false |
| moe_per_layer_logging: false |
| moe_permute_fusion: false |
| moe_router_bias_update_rate: 0.001 |
| moe_router_dtype: null |
| moe_router_enable_expert_bias: false |
| moe_router_group_topk: null |
| moe_router_load_balancing_type: aux_loss |
| moe_router_num_groups: null |
| moe_router_pre_softmax: false |
| moe_router_score_function: softmax |
| moe_router_topk: 2 |
| moe_router_topk_limited_devices: null |
| moe_router_topk_scaling_factor: null |
| moe_shared_expert_intermediate_size: null |
| moe_shared_expert_overlap: false |
| moe_token_dispatcher_type: allgather |
| moe_token_drop_policy: probs |
| moe_token_dropping: false |
| moe_use_legacy_grouped_gemm: false |
| moe_z_loss_coeff: null |
| mtp_loss_scaling_factor: null |
| mtp_num_layers: null |
| multi_latent_attention: false |
| no_sync_func: null |
| normalization: RMSNorm |
| num_attention_heads: 32 |
| num_layers: 52 |
| num_layers_at_end_in_bf16: 1 |
| num_layers_at_start_in_bf16: 1 |
| num_layers_in_first_pipeline_stage: null |
| num_layers_in_last_pipeline_stage: null |
| num_microbatches_with_partial_activation_checkpoints: null |
| num_moe_experts: null |
| num_query_groups: 8 |
| output_layer_init_method: null |
| overlap_p2p_comm: false |
| overlap_p2p_comm_warmup_flush: false |
| parallel_output: true |
| param_sync_func: null |
| params_dtype: |
| _call_: false |
| _target_: torch.bfloat16 |
| perform_initialization: true |
| persist_layer_norm: true |
| pipeline_dtype: null |
| pipeline_model_parallel_comm_backend: null |
| pipeline_model_parallel_size: 1 |
| pipeline_model_parallel_split_rank: null |
| position_embedding_type: none |
| post_process: true |
| pre_process: true |
| qk_layernorm: false |
| recompute_granularity: null |
| recompute_method: null |
| recompute_num_layers: null |
| rotary_base: 10000 |
| rotary_interleaved: false |
| rotary_percent: 1.0 |
| seq_len_interpolation_factor: null |
| seq_length: 8192 |
| sequence_parallel: false |
| share_embeddings_and_output_weights: false |
| softmax_scale: null |
| tensor_model_parallel_size: 1 |
| test_mode: false |
| timers: null |
| tokenizer_library: tiktoken |
| tokenizer_model_path: null |
| tokenizer_name: TiktokenTokenizer |
| tp_comm_atomic_ag: false |
| tp_comm_atomic_rs: false |
| tp_comm_bootstrap_backend: nccl |
| tp_comm_bulk_dgrad: true |
| tp_comm_bulk_wgrad: true |
| tp_comm_overlap: false |
| tp_comm_overlap_ag: true |
| tp_comm_overlap_disable_fc1: false |
| tp_comm_overlap_disable_qkv: false |
| tp_comm_overlap_rs: true |
| tp_comm_overlap_rs_dgrad: false |
| tp_comm_split_ag: true |
| tp_comm_split_rs: true |
| tp_only_amax_red: false |
| use_cpu_initialization: false |
| use_custom_fsdp: false |
| use_ring_exchange_p2p: false |
| use_te_rng_tracker: false |
| variable_seq_lengths: false |
| virtual_pipeline_model_parallel_size: null |
| vocab_file: null |
| vocab_size: 131072 |
| wgrad_deferral_limit: 0 |
| window_size: null |
| model_transform: null |
| optim: |
| _target_: nemo.lightning.pytorch.optim.megatron.MegatronOptimizerModule |
| config: |
| _target_: megatron.core.optimizer.optimizer_config.OptimizerConfig |
| adam_beta1: 0.9 |
| adam_beta2: 0.999 |
| adam_eps: 1.0e-08 |
| barrier_with_L1_time: false |
| bf16: false |
| clip_grad: 1.0 |
| config_logger_dir: '' |
| decoupled_lr: null |
| decoupled_min_lr: null |
| exp_avg_dtype: |
| _call_: false |
| _target_: torch.float32 |
| exp_avg_sq_dtype: |
| _call_: false |
| _target_: torch.float32 |
| fp16: false |
| hysteresis: 2 |
| initial_loss_scale: 4294967296 |
| log_num_zeros_in_grad: false |
| loss_scale: null |
| loss_scale_window: 1000 |
| lr: 0.0001 |
| main_grads_dtype: |
| _call_: false |
| _target_: torch.float32 |
| main_params_dtype: |
| _call_: false |
| _target_: torch.float32 |
| min_loss_scale: 1.0 |
| min_lr: null |
| optimizer: adam |
| optimizer_cpu_offload: false |
| optimizer_offload_fraction: 0.0 |
| overlap_cpu_optimizer_d2h_h2d: false |
| overlap_param_gather_with_optimizer_step: false |
| params_dtype: |
| _call_: false |
| _target_: torch.float32 |
| pin_cpu_grads: true |
| pin_cpu_params: true |
| sgd_momentum: 0.9 |
| timers: null |
| use_distributed_optimizer: true |
| use_precision_aware_optimizer: false |
| use_torch_optimizer_for_cpu_offload: false |
| weight_decay: 0.01 |
| lr_mult: 1.0 |
| lr_scheduler: null |
| no_weight_decay_cond: null |
| scale_lr_cond: null |
| tokenizer: |
| _target_: nemo.collections.common.tokenizers.huggingface.auto_tokenizer.AutoTokenizer |
| additional_special_tokens: [] |
| bos_token: null |
| cls_token: null |
| eos_token: null |
| include_special_tokens: false |
| mask_token: null |
| merges_file: null |
| pad_token: null |
| pretrained_model_name: nemo_tokenizer |
| sep_token: null |
| trust_remote_code: true |
| unk_token: null |
| use_fast: false |
| vocab_file: null |
|
|
