nemo/hparams.yaml

name: gemma-7b-sql-nemo
trainer:
  num_nodes: 1
  devices: 8
  accelerator: gpu
  precision: bf16
  logger: false
  enable_checkpointing: false
  use_distributed_sampler: false
  max_time: null
  max_epochs: 1
  max_steps: -1
  sft:
    max_epochs: 1
    max_steps: -1
    val_check_interval: 1000
    save_interval: 1000
    limit_val_batches: 40
    gradient_clip_val: 1.0
exp_manager:
  explicit_log_dir: models/gemma-7b-sql-nemo
  exp_dir: null
  name: gemma-7b-sql-nemo
  create_wandb_logger: false
  wandb_logger_kwargs:
    project: null
    name: null
  resume_if_exists: true
  resume_ignore_no_checkpoint: true
  create_checkpoint_callback: true
  checkpoint_callback_params:
    monitor: validation_loss
    save_top_k: 5
    mode: min
    save_nemo_on_train_end: true
    filename: megatron_gpt_sft--{validation_loss:.3f}-{step}-{consumed_samples}-{epoch}
    model_parallel_size: 4
    save_best_model: false
model:
  seed: 1234
  tensor_model_parallel_size: 4
  pipeline_model_parallel_size: 1
  restore_from_path: /workspace/models/pytorch-7b-pt.nemo
  resume_from_checkpoint: null
  save_nemo_on_validation_end: true
  sync_batch_comm: false
  megatron_amp_O2: true
  encoder_seq_length: 8192
  sequence_parallel: false
  activations_checkpoint_granularity: null
  activations_checkpoint_method: null
  activations_checkpoint_num_layers: null
  activations_checkpoint_layers_per_pipeline: null
  answer_only_loss: true
  gradient_as_bucket_view: false
  seq_len_interpolation_factor: null
  use_flash_attention: null
  hidden_dropout: 0.0
  attention_dropout: 0.0
  ffn_dropout: 0.0
  peft:
    peft_scheme: none
    restore_from_path: null
    lora_tuning:
      target_modules:
      - attention_qkv
      adapter_dim: 32
      adapter_dropout: 0.0
      column_init_method: xavier
      row_init_method: zero
      layer_selection: null
      weight_tying: false
      position_embedding_strategy: null
  data:
    chat: false
    chat_prompt_tokens:
      system_turn_start: "\0"
      turn_start: "\x11"
      label_start: "\x12"
      end_of_turn: '

        '
      end_of_name: '

        '
    sample: false
    num_workers: 0
    dataloader_type: single
    train_ds:
      file_path: nsql.jsonl
      global_batch_size: 128
      micro_batch_size: 1
      shuffle: true
      memmap_workers: null
      max_seq_length: 8192
      min_seq_length: 1
      drop_last: true
      label_key: output
      add_eos: true
      add_sep: false
      add_bos: false
      truncation_field: input
      index_mapping_dir: null
      prompt_template: '{input} {output}'
      hf_dataset: false
      truncation_method: right
    validation_ds:
      file_path: nsql.jsonl
      global_batch_size: 128
      micro_batch_size: 1
      shuffle: false
      memmap_workers: null
      max_seq_length: 8192
      min_seq_length: 1
      drop_last: true
      label_key: output
      add_eos: true
      add_sep: false
      add_bos: false
      truncation_field: input
      index_mapping_dir: null
      prompt_template: '{input} {output}'
      hf_dataset: false
      truncation_method: right
      output_original_text: true
  optim:
    name: distributed_fused_adam
    lr: 5.0e-06
    weight_decay: 0.01
    betas:
    - 0.9
    - 0.98
    sched:
      name: CosineAnnealing
      warmup_steps: 10
      constant_steps: 1000
      min_lr: 9.0e-07
  bias_activation_fusion: true
  precision: bf16