| adaptor: | |
| encstack_layers: 4 | |
| fuse_layers: | |
| - -1 | |
| - -4 | |
| - -8 | |
| - -12 | |
| fusion_type: tokenwise | |
| num_global_memory_tokens: 16 | |
| type: gated_residual | |
| use_encstack: true | |
| use_global_memory_tokens: true | |
| use_layer_fusion: true | |
| use_salience_gate: true | |
| data: | |
| eval_file: llm2seq/data/processed/vlsp/val.jsonl | |
| max_source_length: 8192 | |
| max_target_length: 768 | |
| source_prefix: "T\xF3m t\u1EAFt c\xE1c v\u0103n b\u1EA3n sau th\xE0nh m\u1ED9t \u0111\ | |
| o\u1EA1n ng\u1EAFn g\u1ECDn:\n" | |
| test_file: llm2seq/data/processed/vlsp/test.jsonl | |
| train_file: llm2seq/data/processed/vlsp/train.jsonl | |
| evaluation: | |
| eval_every_epochs: 1 | |
| metrics: | |
| - rouge | |
| - latency | |
| save_every_epochs: 1 | |
| generation: | |
| do_sample: false | |
| eval_batch_size: 16 | |
| max_new_tokens: 768 | |
| min_new_tokens: 32 | |
| no_repeat_ngram_size: 3 | |
| repetition_penalty: 1.05 | |
| temperature: 0.0 | |
| top_k: 0 | |
| top_p: 1.0 | |
| huggingface: | |
| enabled: true | |
| fail_on_error: true | |
| keep_local_epoch_checkpoints: 1 | |
| path_in_repo: checkpoints/phase2_lora_encoder_vlsp | |
| push_each_epoch: false | |
| push_final_best: true | |
| repo_id: null | |
| repo_type: model | |
| model: | |
| d_dec: 1024 | |
| d_enc: 2048 | |
| encoder_name: McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp | |
| encoder_torch_dtype: bfloat16 | |
| encoder_trainable: true | |
| lora_alpha: 32 | |
| lora_dropout: 0.05 | |
| lora_r: 16 | |
| lora_target_modules: | |
| - q_proj | |
| - k_proj | |
| - v_proj | |
| - o_proj | |
| - gate_proj | |
| - up_proj | |
| - down_proj | |
| use_lora_for_encoder: true | |
| project: | |
| name: llm2seq | |
| output_dir: runs/llm2seq_phase2_lora_encoder_vlsp | |
| task: summarization | |
| small_decoder: | |
| dropout: 0.1 | |
| ffn_size: 4096 | |
| hidden_size: 1024 | |
| max_seq_len: 1024 | |
| num_heads: 16 | |
| num_layers: 8 | |
| tie_embeddings: true | |
| training: | |
| adaptor_lr: 5.0e-05 | |
| batch_size: 2 | |
| bf16: true | |
| decoder_lr: 5.0e-05 | |
| encoder_lr: 0.0001 | |
| fp16: false | |
| grad_accum_steps: 8 | |
| gradient_checkpointing: true | |
| log_every_steps: 2 | |
| max_grad_norm: 1.0 | |
| min_lr_ratio: 0.1 | |
| num_train_epochs: 6 | |
| reset_best_eval_loss_on_resume: true | |
| reset_global_step_on_resume: true | |
| seed: 42 | |
| skip_optimizer_resume: true | |
| stage: phase2_lora_encoder_vlsp | |
| tf32: true | |
| warmup_ratio: 0.05 | |
| weight_decay: 0.01 | |