model_config.yaml

mcore_gpt: true
micro_batch_size: 1
global_batch_size: 256
tensor_model_parallel_size: 8
pipeline_model_parallel_size: 4
virtual_pipeline_model_parallel_size: null
encoder_seq_length: 4096
max_position_embeddings: 4096
num_layers: 96
hidden_size: 18432
ffn_hidden_size: 73728
num_attention_heads: 96
init_method_std: 0.0063
use_scaled_init_method: true
hidden_dropout: 0.0
attention_dropout: 0.0
ffn_dropout: 0.0
kv_channels: null
apply_query_key_layer_scaling: true
normalization: layernorm1p
layernorm_epsilon: 1.0e-05
do_layer_norm_weight_decay: false
make_vocab_size_divisible_by: 128
pre_process: true
post_process: true
persist_layer_norm: true
bias: false
activation: squared-relu
headscale: false
transformer_block_type: pre_ln
openai_gelu: false
normalize_attention_scores: true
position_embedding_type: rope
rotary_percentage: 0.5
attention_type: multihead
share_embeddings_and_output_weights: false
num_query_groups: 8
tokenizer:
  library: sentencepiece
  type: null
  model: nemo:8223bf8eaa194eb8920af568bb52e2d0_megatron_2.model
  vocab_file: null
  merge_file: null
  tokenizer_model: nemo:eb5528fdec5c4083affa2c97958eeef7_megatron_2.model
  sentencepiece_legacy: false
native_amp_init_scale: 4294967296
native_amp_growth_interval: 1000
hysteresis: 2
fp32_residual_connection: false
fp16_lm_cross_entropy: false
megatron_amp_O2: true
grad_allreduce_chunk_size_mb: 125
grad_div_ar_fusion: true
gradient_accumulation_fusion: false
bias_activation_fusion: false
bias_dropout_add_fusion: false
masked_softmax_fusion: true
seed: 1234
resume_from_checkpoint: null
use_cpu_initialization: false
onnx_safe: false
apex_transformer_log_level: 30
gradient_as_bucket_view: false
sync_batch_comm: false
activations_checkpoint_granularity: null
activations_checkpoint_method: null
activations_checkpoint_num_layers: 1
num_micro_batches_with_partial_activation_checkpoints: null
activations_checkpoint_layers_per_pipeline: null
sequence_parallel: false
transformer_engine: false
fp8: false
fp8_e4m3: false
fp8_hybrid: false
fp8_margin: 0
fp8_interval: 1
fp8_amax_history_len: 1
fp8_amax_compute_algo: most_recent
reduce_amax: true
use_emha: false
optim:
  name: distributed_fused_adam
  lr: 3.002e-07
  weight_decay: 0.1
  betas:
  - 0.9
  - 0.98
  sched:
    name: CosineAnnealing
    warmup_steps: 10
    constant_steps: 400
    min_lr: 3.0e-07
  bucket_cap_mb: 200
  overlap_grad_sync: false
  contiguous_grad_buffer: true
precision: bf16-mixed
data:
  chat: true
  chat_prompt_tokens:
    system_turn_start: <extra_id_0>
    turn_start: <extra_id_1>
    label_start: <extra_id_2>
    end_of_turn: '

      '
    end_of_name: '

      '
  sample: true
  num_workers: 2
  dataloader_type: single
  train_ds:
    file_path: /dataset/daring-anteater_commercial.shuf.removelong.jsonl
    global_batch_size: 128
    micro_batch_size: 1
    shuffle: true
    memmap_workers: null
    max_seq_length: 4096
    min_seq_length: 1
    drop_last: true
    concat_sampling_probabilities: null
    label_key: output
    add_eos: false
    add_sep: false
    add_bos: false
    truncation_field: input
    index_mapping_dir: /indexmap_dir
    prompt_template: '<extra_id_0>System

      {system message}

      <extra_id_1>User

      {turn 1 user message}

      <extra_id_1>Assistant

      <extra_id_2>{turn 1 assistant label}

      {turn 1 assistant message}

      <extra_id_1>User

      {turn 2 user message}

      <extra_id_1>Assistant

      <extra_id_2>{turn 2 assistant label}

      {turn 2 assistant message}

      <extra_id_1>'
    hf_dataset: true
    truncation_method: right
  validation_ds:
    file_path: /dataset/daring-anteater_commercial.shuf.removelong.jsonl
    names: null
    global_batch_size: 128
    micro_batch_size: 1
    shuffle: false
    memmap_workers: null
    max_seq_length: 4096
    min_seq_length: 1
    drop_last: false
    label_key: output
    add_eos: false
    add_sep: false
    add_bos: false
    write_predictions_to_file: false
    output_file_path_prefix: null
    truncation_field: input
    index_mapping_dir: /indexmap_dir
    prompt_template: '<extra_id_0>System

      {system message}

      <extra_id_1>User

      {turn 1 user message}

      <extra_id_1>Assistant

      <extra_id_2>{turn 1 assistant label}

      {turn 1 assistant message}

      <extra_id_1>User

      {turn 2 user message}

      <extra_id_1>Assistant

      <extra_id_2>{turn 2 assistant label}

      {turn 2 assistant message}

      <extra_id_1>'
    tokens_to_generate: 32
    hf_dataset: true
    truncation_method: right
    metric:
      name: loss
      average: null
      num_classes: null
  test_ds:
    prompt_template: '<extra_id_0>System

      {system message}

      <extra_id_1>User

      {turn 1 user message}

      <extra_id_1>Assistant

      <extra_id_2>{turn 1 assistant label}

      {turn 1 assistant message}

      <extra_id_1>User

      {turn 2 user message}

      <extra_id_1>Assistant

      <extra_id_2>{turn 2 assistant label}

      {turn 2 assistant message}

      <extra_id_1>'
  data_impl: jsonl
  splits_string: null
  seq_length: 4096
  skip_warmup: true
  reset_position_ids: false
  reset_attention_mask: false
  eod_mask_loss: false
  index_mapping_dir: /indexmap_dir
  data_prefix:
    train:
    - /datasets/v30_benign-walrus_clip153600.jsonl
    validation:
    - /datasets/v30_benign-walrus_clip153600.jsonl
    test:
    - /datasets/v30_benign-walrus_clip153600.jsonl
answer_only_loss: true
restore_from_path: /models/340B_100p_CT_100B
save_nemo_on_validation_end: true
use_flash_attention: null
pipeline_model_parallel_split_rank: 0
dpo:
  log_prob_forward_micro_batch_size: 2
  ref_policy_kl_penalty: 0.3
  average_log_probs: false
  sft_loss_coeff: 1.0e-05
  optimize_ref_policy_kl_penalty: false
  preference_loss: reward_rev_dpo
  gt_reward_scale: 1.0
apply_rope_fusion: false
target: nemo.collections.nlp.models.language_modeling.megatron_gpt_model.MegatronGPTModel
nemo_version: 1.22.0