adaptor:
  encstack_layers: 4
  fuse_layers:
  - -1
  - -4
  - -8
  - -12
  fusion_type: tokenwise
  num_global_memory_tokens: 16
  type: gated_residual
  use_encstack: true
  use_global_memory_tokens: true
  use_layer_fusion: true
  use_salience_gate: true
data:
  eval_file: llm2seq/data/processed/vlsp/val.jsonl
  max_source_length: 8192
  max_target_length: 768
  source_prefix: "T\xF3m t\u1EAFt c\xE1c v\u0103n b\u1EA3n sau th\xE0nh m\u1ED9t \u0111\
    o\u1EA1n ng\u1EAFn g\u1ECDn:\n"
  test_file: llm2seq/data/processed/vlsp/test.jsonl
  train_file: llm2seq/data/processed/vlsp/train.jsonl
evaluation:
  eval_every_epochs: 1
  metrics:
  - rouge
  - latency
  save_every_epochs: 1
generation:
  do_sample: false
  eval_batch_size: 16
  max_new_tokens: 768
  min_new_tokens: 32
  no_repeat_ngram_size: 3
  repetition_penalty: 1.05
  temperature: 0.0
  top_k: 0
  top_p: 1.0
huggingface:
  enabled: true
  fail_on_error: true
  keep_local_epoch_checkpoints: 1
  path_in_repo: checkpoints/phase2_lora_encoder_vlsp
  push_each_epoch: false
  push_final_best: true
  repo_id: null
  repo_type: model
model:
  d_dec: 1024
  d_enc: 2048
  encoder_name: McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp
  encoder_torch_dtype: bfloat16
  encoder_trainable: true
  lora_alpha: 32
  lora_dropout: 0.05
  lora_r: 16
  lora_target_modules:
  - q_proj
  - k_proj
  - v_proj
  - o_proj
  - gate_proj
  - up_proj
  - down_proj
  use_lora_for_encoder: true
project:
  name: llm2seq
  output_dir: runs/llm2seq_phase2_lora_encoder_vlsp
  task: summarization
small_decoder:
  dropout: 0.1
  ffn_size: 4096
  hidden_size: 1024
  max_seq_len: 1024
  num_heads: 16
  num_layers: 8
  tie_embeddings: true
training:
  adaptor_lr: 5.0e-05
  batch_size: 2
  bf16: true
  decoder_lr: 5.0e-05
  encoder_lr: 0.0001
  fp16: false
  grad_accum_steps: 8
  gradient_checkpointing: true
  log_every_steps: 2
  max_grad_norm: 1.0
  min_lr_ratio: 0.1
  num_train_epochs: 6
  reset_best_eval_loss_on_resume: true
  reset_global_step_on_resume: true
  seed: 42
  skip_optimizer_resume: true
  stage: phase2_lora_encoder_vlsp
  tf32: true
  warmup_ratio: 0.05
  weight_decay: 0.01