File size: 4,230 Bytes
c9c6765 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 |
from unsloth import PatchDPOTrainer # This line is from the DPO Zephyr example ******
PatchDPOTrainer()
from huggingface_hub import HfApi
from huggingface_hub import create_repo
from unsloth import FastLanguageModel
import torch
from datasets import load_dataset
import random
max_seq_length = 4096 # Choose any! We auto support RoPE Scaling internally!
dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
repo_name = "dpo-v1-Nemo"
# do wandb stuff
import wandb
import random
wandb.init(
project="huggingface",
name= repo_name,)
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "ijic062/Nemo-v1.1",
max_seq_length = max_seq_length,
dtype = dtype,
load_in_4bit = load_in_4bit,
token = "", # use one if using gated models like meta-llama/Llama-2-7b-hf
)
########################################################################################################
model = FastLanguageModel.get_peft_model(
model,
r = 64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
"gate_proj", "up_proj", "down_proj",],
lora_alpha = 16,
lora_dropout = 0, # Supports any, but = 0 is optimized
bias = "none", # Supports any, but = "none" is optimized
# [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
random_state = 3407,
use_rslora = False, # We support rank stabilized LoRA
loftq_config = None, # And LoftQ
)
######################################################################################################### ***
dataset = load_dataset(
"Chaser-cz/dpo-nice-prompt"
)
train_dataset = dataset['train'].shuffle(seed=random.randint(1, 9999))
# Shuffles data and take a small portion
# test_dataset = dataset['test_prefs']
column_names = list(dataset["train"].features)
print(f"This is column names: {column_names}")
import pprint
row = train_dataset[9]
pprint.pprint(row["prompt"])
pprint.pprint(row["chosen"])
pprint.pprint(row["rejected"])
##########################################################################################################
from unsloth import PatchDPOTrainer
PatchDPOTrainer()
from trl import DPOTrainer
from transformers import TrainingArguments
from unsloth import is_bfloat16_supported
dpo_trainer = DPOTrainer(
model = model,
beta = 0.5,
tokenizer = tokenizer,
max_length = 1024,
max_prompt_length = 512,
train_dataset = train_dataset,
ref_model = None,
# dataset_text_field = "text",
# max_seq_length = max_seq_length,
# dataset_num_proc = 2,
# packing = False, # Can make training 5x faster for short sequences.
args = TrainingArguments(
# loss_type = "sigmoid",
per_device_train_batch_size = 2,
gradient_accumulation_steps = 32,
gradient_checkpointing= True,
warmup_steps = 5,
#num_train_epochs = 3,
max_steps = 1000,
learning_rate = 2.5e-4,
fp16 = not is_bfloat16_supported(),
bf16 = is_bfloat16_supported(),
logging_steps = 1,
optim = "adamw_8bit",
weight_decay = 0.07,
lr_scheduler_type = "cosine",
seed = 3407,
output_dir = "outputs/dpo-out-13b",
save_strategy = "steps",
save_steps = 500,
),
)
dpo_trainer.train()
########################################################################################################### ***
model.save_pretrained_merged("outputs/dpo-out-13b/merged", tokenizer, save_method = "merged_16bit")
api = HfApi()
create_repo(f"jic062/{repo_name}", repo_type="model",private=True,token="")
api.upload_folder(
folder_path="outputs/dpo-out-13b/merged",
repo_id=f"jic062/{repo_name}",
repo_type="model",
)
wandb.finish()
|