| encoder: |
| num_layers: 8 |
| hidden_size: 512 |
| ffn_hidden_size: 1024 |
| num_attention_heads: 6 |
| init_method_std: 0.02 |
| hidden_dropout: 0.0 |
| attention_dropout: 0.0 |
| ffn_dropout: 0.0 |
| position_embedding_type: relative |
| relative_attention_num_buckets: 32 |
| relative_attention_max_distance: 128 |
| relative_position_bias_self_attention_only: true |
| kv_channels: 64 |
| apply_query_key_layer_scaling: false |
| layernorm_epsilon: 1.0e-06 |
| persist_layer_norm: true |
| bias_activation_fusion: false |
| grad_div_ar_fusion: true |
| masked_softmax_fusion: false |
| bias_dropout_add_fusion: false |
| bias: false |
| normalization: rmsnorm |
| arch: transformer |
| activation: geglu |
| headscale: false |
| transformer_block_type: pre_ln |
| hidden_steps: 32 |
| num_self_attention_per_cross_attention: 1 |
| openai_gelu: true |
| onnx_safe: false |
| fp32_residual_connection: false |
| activations_checkpoint_method: null |
| activations_checkpoint_num_layers: 1 |
| megatron_legacy: true |
| normalize_attention_scores: false |
| decoder: |
| num_layers: 8 |
| hidden_size: 512 |
| ffn_hidden_size: 1024 |
| num_attention_heads: 6 |
| init_method_std: 0.02 |
| hidden_dropout: 0.0 |
| attention_dropout: 0.0 |
| ffn_dropout: 0.0 |
| position_embedding_type: relative |
| relative_attention_num_buckets: 32 |
| relative_attention_max_distance: 128 |
| relative_position_bias_self_attention_only: true |
| kv_channels: 64 |
| apply_query_key_layer_scaling: false |
| layernorm_epsilon: 1.0e-06 |
| persist_layer_norm: true |
| bias_activation_fusion: false |
| grad_div_ar_fusion: true |
| masked_softmax_fusion: false |
| bias_dropout_add_fusion: false |
| bias: false |
| normalization: rmsnorm |
| arch: transformer |
| activation: geglu |
| headscale: false |
| transformer_block_type: pre_ln |
| hidden_steps: 32 |
| num_self_attention_per_cross_attention: 1 |
| openai_gelu: true |
| onnx_safe: false |
| fp32_residual_connection: false |
| activations_checkpoint_method: null |
| activations_checkpoint_num_layers: 1 |
| megatron_legacy: true |
| normalize_attention_scores: false |
| micro_batch_size: 4 |
| global_batch_size: 8 |
| tensor_model_parallel_size: 1 |
| pipeline_model_parallel_size: 1 |
| resume_from_checkpoint: null |
| pipeline_model_parallel_split_rank: 0 |
| make_vocab_size_divisible_by: 128 |
| megatron_amp_O2: false |
| grad_allreduce_chunk_size_mb: 125 |
| grad_div_ar_fusion: true |
| gradient_as_bucket_view: true |
| seq_length: 512 |
| max_position_embeddings: 512 |
| tokenizer: |
| library: sentencepiece |
| type: null |
| model: nemo:ce65b6d8f4fb4975955e935db699cba3_t5_small_tokenizer.model |
| vocab_file: null |
| merge_file: null |
| num_sentinel_tokens: 100 |
| sentencepiece_legacy: true |
| add_sentinel_tokens_in_reverse_order: true |
| add_sentinel_tokens_first: true |
| embedding_init_method_std: 0.02 |
| embedding_dropout: 0.1 |
| share_token_embeddings: true |
| share_decoder_tokens_head_embeddings: false |
| tokens_head_bias: false |
| native_amp_init_scale: 4294967296 |
| native_amp_growth_interval: 1000 |
| fp16_lm_cross_entropy: false |
| seed: 1234 |
| use_cpu_initialization: false |
| apex_transformer_log_level: 30 |
| data: |
| data_prefix: null |
| index_mapping_dir: null |
| data_impl: mmap |
| splits_string: 949,45,5 |
| seq_length: 512 |
| seq_length_dec: 128 |
| skip_warmup: true |
| num_workers: 0 |
| dataloader_type: single |
| masked_lm_prob: 0.15 |
| dataset_type: t5 |
| short_seq_prob: 0.0 |
| max_ngram_size: 10 |
| mean_ngram_size: null |
| geometric_dist: true |
| permutation: false |
| whole_word_masking: false |
| favor_longer_ngrams: false |
| respect_document_boundaries: true |
| optim: |
| name: fused_adam |
| lr: 0.0001 |
| betas: |
| - 0.9 |
| - 0.999 |
| eps: 1.0e-08 |
| weight_decay: 0.01 |
| sched: |
| name: WarmupAnnealing |
| min_lr: 1.0e-05 |
| last_epoch: -1 |
| warmup_ratio: 0.01 |
| precision: bf16 |
| target: nemo.collections.nlp.models.language_modeling.megatron_t5_model.MegatronT5Model |
| nemo_version: 1.11.0rc0 |
| library: huggingface-t5v1_1 |
