sinergiaepc/llama2-13b_2024-04-16

First version of the fine tuned llama 2 13B.

Trained on Sinergia's workstation (mounting a Nvidia RTX 4080 super).

Train configuration:

model_name: "NousResearch/Llama-2-13b-chat-hf"
dataset_name: "sinergiaepc/Insta360_pro2_2024_04_05"
new_model: "sinergiaepc/llama2-7b_2024-04-16"
qlora_parameters:
  r: 64
  lora_alpha: 16
  lora_dropout: 0.1
  bias: "none"
  task_type: "CAUSAL_LM"
bitsandbytes_parameters:
  load_in_4bit: true
  bnb_4bit_compute_dtype: "float16"
  bnb_4bit_quant_type: "nf4"
  bnb_4bit_use_double_quant: false
training_arguments:
  output_dir: "./results"
  num_train_epochs: 20
  fp16: false
  bf16: false
  per_device_train_batch_size: 1
  # per_device_eval_batch_size: 1
  gradient_accumulation_steps: 16
  # gradient_checkpointing: true
  max_grad_norm: 0.3
  learning_rate: 0.0002
  weight_decay: 0.001
  optim: "paged_adamw_32bit"
  lr_scheduler_type: "cosine"
  max_steps: -1
  warmup_ratio: 0.03
  group_by_length: true
  save_steps: 0
  logging_steps: 25
  # report_to: "tensorboard"
sft_parameters:
  max_seq_length: null
  packing: false