File size: 3,533 Bytes
ec1110c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 |
import torch
from huggingface_hub import HfApi
from huggingface_hub import create_repo
from unsloth import FastLanguageModel
import torch
from datasets import load_dataset
import random
max_seq_length = 2048
dtype = None
load_in_4bit = True
repo_name = "instruct-v19"
# do wandb stuff
import wandb
wandb.init(
project="unsloth_lora",
name= repo_name,
)
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct", #mistralai/Mistral-Nemo-Instruct-2407
max_seq_length = max_seq_length,
dtype = dtype,
load_in_4bit = load_in_4bit,
token = "", # use one if using gated models like meta-llama/Llama-2-7b-hf
)
model = FastLanguageModel.get_peft_model(
model,
r = 64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
"gate_proj", "up_proj", "down_proj",],
lora_alpha = 16,
lora_dropout = 0, # Supports any, but = 0 is optimized
bias = "none", # Supports any, but = "none" is optimized
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
random_state = 3407,
use_rslora = False, # We support rank stabilized LoRA
loftq_config = None, # And LoftQ
)
from datasets import load_dataset
dataset = load_dataset("Chaser-cz/ChaiTop100-SHAREGPT")
train_dataset = dataset["train"].shuffle(seed=random.randint(1, 9999))
from unsloth.chat_templates import get_chat_template
tokenizer = get_chat_template(
tokenizer,
chat_template = "llama-3",
mapping = {"role" : "from", "content" : "value", "user" : "human", "assistant" : "gpt"}, # ShareGPT style
)
def formatting_prompts_func(examples):
convos = examples["conversations"]
texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False) for convo in convos]
return { "text" : texts, }
pass
train_dataset = train_dataset.map(formatting_prompts_func, batched = True,)
from trl import SFTTrainer
from transformers import TrainingArguments
from unsloth import is_bfloat16_supported
trainer = SFTTrainer(
model = model,
tokenizer = tokenizer,
train_dataset = train_dataset,
dataset_text_field = "text",
max_seq_length = max_seq_length,
dataset_num_proc = 2,
packing = False, # Can make training 5x faster for short sequences.
args = TrainingArguments(
per_device_train_batch_size = 2,
gradient_accumulation_steps = 32,
warmup_steps = 5,
max_steps = 1000,
learning_rate = 2.5e-4,
fp16 = not is_bfloat16_supported(),
bf16 = is_bfloat16_supported(),
logging_steps = 1,
optim = "adamw_8bit",
weight_decay = 0.01,
lr_scheduler_type = "cosine",
seed = 3407,
output_dir = "outputs/lora-out-8b",
save_strategy = "steps",
save_steps = 500,)
)
trainer_stats = trainer.train()
model.save_pretrained_merged("outputs/lora-out-8b/merged", tokenizer, save_method = "merged_16bit",)
api = HfApi()
create_repo(f"jic062/{repo_name}", repo_type="model",private=True, token="")
api.upload_folder(
folder_path="outputs/lora-out-8b/merged",
repo_id=f"jic062/{repo_name}",
repo_type="model",
)
wandb.finish()
|