| mcore_gpt: true |
| micro_batch_size: 1 |
| global_batch_size: 2308 |
| tensor_model_parallel_size: 8 |
| pipeline_model_parallel_size: 2 |
| virtual_pipeline_model_parallel_size: null |
| encoder_seq_length: 4096 |
| max_position_embeddings: 4096 |
| num_layers: 96 |
| hidden_size: 18432 |
| ffn_hidden_size: 73728 |
| num_attention_heads: 96 |
| init_method_std: 0.0063 |
| use_scaled_init_method: true |
| hidden_dropout: 0.0 |
| attention_dropout: 0.0 |
| ffn_dropout: 0.0 |
| kv_channels: null |
| apply_query_key_layer_scaling: true |
| normalization: layernorm1p |
| layernorm_epsilon: 1.0e-05 |
| do_layer_norm_weight_decay: false |
| make_vocab_size_divisible_by: 128 |
| pre_process: true |
| post_process: true |
| persist_layer_norm: true |
| bias: false |
| activation: squared-relu |
| headscale: false |
| transformer_block_type: pre_ln |
| openai_gelu: false |
| normalize_attention_scores: true |
| position_embedding_type: rope |
| rotary_percentage: 0.5 |
| attention_type: multihead |
| share_embeddings_and_output_weights: false |
| num_query_groups: 8 |
| tokenizer: |
| library: sentencepiece |
| type: null |
| model: nemo:d6b0ba93e9734b138f5fc61f5652efbd_nemotron_2_256k.model |
| delimiter: null |
| vocab_file: null |
| merge_file: null |
| sentencepiece_legacy: false |
| tokenizer_model: nemo:29e0db5f7dd14bcf9f32727ff482502b_nemotron_2_256k.model |
| native_amp_init_scale: 4294967296 |
| native_amp_growth_interval: 1000 |
| hysteresis: 2 |
| fp32_residual_connection: false |
| fp16_lm_cross_entropy: false |
| megatron_amp_O2: true |
| grad_allreduce_chunk_size_mb: 125 |
| grad_div_ar_fusion: true |
| gradient_accumulation_fusion: false |
| bias_activation_fusion: false |
| bias_dropout_add_fusion: false |
| masked_softmax_fusion: true |
| seed: 1234 |
| resume_from_checkpoint: null |
| use_cpu_initialization: false |
| onnx_safe: false |
| apex_transformer_log_level: 30 |
| gradient_as_bucket_view: true |
| sync_batch_comm: false |
| activations_checkpoint_granularity: null |
| activations_checkpoint_method: null |
| activations_checkpoint_num_layers: null |
| num_micro_batches_with_partial_activation_checkpoints: null |
| activations_checkpoint_layers_per_pipeline: null |
| sequence_parallel: true |
| transformer_engine: false |
| fp8: false |
| fp8_e4m3: false |
| fp8_hybrid: false |
| fp8_margin: 0 |
| fp8_interval: 1 |
| fp8_amax_history_len: 1 |
| fp8_amax_compute_algo: most_recent |
| reduce_amax: true |
| use_emha: false |
| optim: |
| name: distributed_fused_adam |
| lr: '1.0e-4' |
| weight_decay: 0.1 |
| betas: |
| - 0.9 |
| - 0.95 |
| sched: |
| name: CosineAnnealing |
| warmup_steps: 500 |
| constant_steps: 50000 |
| min_lr: '1.0e-7' |
| apply_rope_fusion: false |
| precision: bf16 |
| target: nemo.collections.nlp.models.language_modeling.megatron_gpt_model.MegatronGPTModel |
| nemo_version: 1.15.0rc0 |
|
|
