| | --- |
| | license: gemma |
| | tags: |
| | - unsloth |
| | - trl |
| | - sft |
| | --- |
| | |
| | Experiment 1 SFT ALPACA INDO |
| |
|
| | dataset: 9 millions token indo alpaca dataset |
| |
|
| | max_seq_length = 8192, |
| | dataset_num_proc = 2, |
| | packing = False, |
| | args = TrainingArguments( |
| | per_device_train_batch_size = 1, |
| | gradient_accumulation_steps = 8, |
| | warmup_steps = 5, |
| | num_train_epochs = 1, |
| | learning_rate = 5e-5, |
| | fp16 = not is_bfloat16_supported(), |
| | bf16 = is_bfloat16_supported(), |
| | logging_steps = 1, |
| | optim = "adamw_8bit", |
| | weight_decay = 0.01, |
| | lr_scheduler_type = "linear", |
| | seed = 3407, |
