Add training config

config/llama31_hatexplain_qlora_sft.yaml

@@ -0,0 +1,50 @@
+### model
+model_name_or_path: meta-llama/Meta-Llama-3.1-8B-Instruct
+quantization_bit: 4          # QLoRA backbone (bnb 4-bit)
+quantization_method: bnb     # choices: [bnb, hqq, eetq]
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+do_eval: true
+finetuning_type: lora        # LoRA on quantized base = QLoRA
+use_dora: false              # DoRA is not compatible with PTQ/QLoRA here
+lora_rank: 8
+lora_alpha: 16
+lora_dropout: 0.05
+lora_target: all
+gradient_checkpointing: true
+
+### dataset
+dataset: hatexplain_rationale_train
+eval_dataset: hatexplain_validation
+template: llama3
+cutoff_len: 1024
+overwrite_cache: true
+preprocessing_num_workers: 8
+dataloader_num_workers: 2
+
+### output
+output_dir: saves/llama31-8b/hatexplain_rationale/qlora
+logging_steps: 10
+save_steps: 500
+eval_steps: 500
+save_only_model: true
+overwrite_output_dir: true
+report_to: none  # [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 3   # increase if memory allows
+gradient_accumulation_steps: 8
+learning_rate: 2.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+per_device_eval_batch_size: 1
+eval_strategy: steps
+