Upload dpo-train.py with huggingface_hub

dpo-train.py

@@ -0,0 +1,122 @@
+from unsloth import PatchDPOTrainer   # This line is from the DPO Zephyr example ******
+PatchDPOTrainer()
+from huggingface_hub import HfApi
+from huggingface_hub import create_repo
+from unsloth import FastLanguageModel
+import torch
+from datasets import load_dataset
+import random
+
+max_seq_length = 4096 # Choose any! We auto support RoPE Scaling internally!
+dtype = None # None for auto detection. Float16 for Tesla T4, V100, Bfloat16 for Ampere+
+load_in_4bit = True # Use 4bit quantization to reduce memory usage. Can be False.
+repo_name = "dpo-v1-Nemo"
+# do wandb stuff
+import wandb
+import random
+wandb.init(
+        project="huggingface",
+        name= repo_name,)
+
+
+model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name = "ijic062/Nemo-v1.1", 
+        max_seq_length = max_seq_length,
+        dtype = dtype,
+        load_in_4bit = load_in_4bit,
+        token = "", # use one if using gated models like meta-llama/Llama-2-7b-hf
+)
+
+########################################################################################################
+
+model = FastLanguageModel.get_peft_model(
+
+        model,
+        r = 64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+        target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
+                        "gate_proj", "up_proj", "down_proj",],
+        lora_alpha = 16,
+        lora_dropout = 0, # Supports any, but = 0 is optimized
+        bias = "none",    # Supports any, but = "none" is optimized
+        # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
+        use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
+        random_state = 3407,
+        use_rslora = False,  # We support rank stabilized LoRA
+        loftq_config = None, # And LoftQ
+
+)
+
+#########################################################################################################           ***
+
+dataset = load_dataset(
+    "Chaser-cz/dpo-nice-prompt"
+)
+
+train_dataset = dataset['train'].shuffle(seed=random.randint(1, 9999))
+
+# Shuffles data and take a small portion
+# test_dataset = dataset['test_prefs']
+
+column_names = list(dataset["train"].features)
+print(f"This is column names: {column_names}")
+
+import pprint
+row = train_dataset[9]
+pprint.pprint(row["prompt"])
+pprint.pprint(row["chosen"])
+pprint.pprint(row["rejected"])
+##########################################################################################################
+
+from unsloth import PatchDPOTrainer
+PatchDPOTrainer()
+from trl import DPOTrainer
+from transformers import TrainingArguments
+from unsloth import is_bfloat16_supported
+
+dpo_trainer = DPOTrainer(
+        model = model,
+        beta = 0.5,
+        tokenizer = tokenizer,
+        max_length = 1024,
+        max_prompt_length = 512,
+        train_dataset = train_dataset,
+        ref_model = None,
+        # dataset_text_field = "text",
+        # max_seq_length = max_seq_length,
+        # dataset_num_proc = 2,
+        # packing = False, # Can make training 5x faster for short sequences.
+        args = TrainingArguments(
+            # loss_type = "sigmoid",
+            per_device_train_batch_size = 2,
+            gradient_accumulation_steps = 32,
+            gradient_checkpointing= True, 
+            warmup_steps = 5,
+            #num_train_epochs = 3,
+            max_steps = 1000,
+            learning_rate = 2.5e-4,
+            fp16 = not is_bfloat16_supported(),
+            bf16 = is_bfloat16_supported(),
+            logging_steps = 1,
+            optim = "adamw_8bit",
+            weight_decay = 0.07,
+            lr_scheduler_type = "cosine",
+            seed = 3407,
+            output_dir = "outputs/dpo-out-13b",
+            save_strategy = "steps",
+            save_steps = 500,
+        ),
+)
+
+dpo_trainer.train()
+
+###########################################################################################################      ***
+model.save_pretrained_merged("outputs/dpo-out-13b/merged", tokenizer, save_method = "merged_16bit")
+api = HfApi()
+create_repo(f"jic062/{repo_name}", repo_type="model",private=True,token="")
+api.upload_folder(        
+        folder_path="outputs/dpo-out-13b/merged",
+        repo_id=f"jic062/{repo_name}",
+        repo_type="model",
+)
+wandb.finish()
+