---
library_name: peft
base_model: athirdpath/BigMistral-13b
---
base_model: athirdpath/BigMistral-13b

model_type: MistralForCausalLM

tokenizer_type: LlamaTokenizer

is_mistral_derived_model: true

load_in_4bit: true

datasets:

  - path: glueLORA2.jsonl
  - 
    type: alpaca

val_set_size: 0.07

adapter: qlora

sequence_len: 4096

sample_packing: true

pad_to_sequence_len: true

lora_r: 512

lora_alpha: 32

lora_dropout: 0.04

lora_target_linear: true

gradient_accumulation_steps: 6

micro_batch_size: 3

eval_batch_size: 3

num_epochs: 4

optimizer: adamw_bnb_8bit

lr_scheduler: cosine

learning_rate: 0.00005

bf16: true

gradient_checkpointing: true

flash_attention: true

warmup_steps: 10

weight_decay: 0.00001