| | --- |
| | library_name: peft |
| | base_model: athirdpath/BigMistral-13b |
| | --- |
| | base_model: athirdpath/BigMistral-13b |
| | |
| | model_type: MistralForCausalLM |
| |
|
| | tokenizer_type: LlamaTokenizer |
| | |
| | is_mistral_derived_model: true |
| |
|
| | load_in_4bit: true |
| |
|
| | datasets: |
| |
|
| | - path: glueLORA2.jsonl |
| | - |
| | type: alpaca |
| |
|
| | val_set_size: 0.07 |
| |
|
| | adapter: qlora |
| |
|
| | sequence_len: 4096 |
| | |
| | sample_packing: true |
| |
|
| | pad_to_sequence_len: true |
| | |
| | lora_r: 512 |
| |
|
| | lora_alpha: 32 |
| | |
| | lora_dropout: 0.04 |
| |
|
| | lora_target_linear: true |
| |
|
| | gradient_accumulation_steps: 6 |
| |
|
| | micro_batch_size: 3 |
| |
|
| | eval_batch_size: 3 |
| |
|
| | num_epochs: 4 |
| | |
| | optimizer: adamw_bnb_8bit |
| | |
| | lr_scheduler: cosine |
| |
|
| | learning_rate: 0.00005 |
| | |
| | bf16: true |
| | |
| | gradient_checkpointing: true |
| |
|
| | flash_attention: true |
| | |
| | warmup_steps: 10 |
| |
|
| | weight_decay: 0.00001 |
