Upload train.py with huggingface_hub

train.py

@@ -0,0 +1,105 @@
+import torch
+from huggingface_hub import HfApi
+from huggingface_hub import create_repo
+from unsloth import FastLanguageModel
+import torch
+from datasets import load_dataset
+import random
+
+max_seq_length = 2048
+dtype = None
+load_in_4bit = True
+repo_name = "instruct-v19"
+# do wandb stuff
+import wandb
+wandb.init(
+        project="unsloth_lora",
+        name= repo_name,
+)
+
+model, tokenizer = FastLanguageModel.from_pretrained(
+
+        model_name = "meta-llama/Meta-Llama-3.1-8B-Instruct", #mistralai/Mistral-Nemo-Instruct-2407
+        max_seq_length = max_seq_length,
+        dtype = dtype,
+        load_in_4bit = load_in_4bit,
+        token = "", # use one if using gated models like meta-llama/Llama-2-7b-hf
+)
+
+model = FastLanguageModel.get_peft_model(
+
+        model,
+        r = 64, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+        target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
+                          "gate_proj", "up_proj", "down_proj",],
+        lora_alpha = 16,
+        lora_dropout = 0, # Supports any, but = 0 is optimized
+        bias = "none",    # Supports any, but = "none" is optimized
+        # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
+        use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
+        random_state = 3407,
+        use_rslora = False,  # We support rank stabilized LoRA
+        loftq_config = None, # And LoftQ
+)
+
+from datasets import load_dataset
+dataset = load_dataset("Chaser-cz/ChaiTop100-SHAREGPT")
+train_dataset = dataset["train"].shuffle(seed=random.randint(1, 9999))
+from unsloth.chat_templates import get_chat_template
+tokenizer = get_chat_template(
+    tokenizer,
+    chat_template = "llama-3",
+    mapping = {"role" : "from", "content" : "value", "user" : "human", "assistant" : "gpt"}, # ShareGPT style
+)
+def formatting_prompts_func(examples):
+    convos = examples["conversations"]
+    texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False) for convo in convos] 
+    return { "text" : texts, }
+pass
+
+train_dataset = train_dataset.map(formatting_prompts_func, batched = True,)
+from trl import SFTTrainer
+from transformers import TrainingArguments
+from unsloth import is_bfloat16_supported
+
+trainer = SFTTrainer(
+
+        model = model,
+        tokenizer = tokenizer,
+        train_dataset = train_dataset,
+        dataset_text_field = "text",
+        max_seq_length = max_seq_length,
+        dataset_num_proc = 2,
+        packing = False, # Can make training 5x faster for short sequences.
+        args = TrainingArguments(
+
+            per_device_train_batch_size = 2,
+            gradient_accumulation_steps = 32,
+            warmup_steps = 5,
+            max_steps = 1000,
+            learning_rate = 2.5e-4,
+            fp16 = not is_bfloat16_supported(),
+            bf16 = is_bfloat16_supported(),
+            logging_steps = 1,
+            optim = "adamw_8bit",
+            weight_decay = 0.01,
+            lr_scheduler_type = "cosine",
+            seed = 3407,
+            output_dir = "outputs/lora-out-8b",
+            save_strategy = "steps",
+            save_steps = 500,)
+)
+
+
+trainer_stats = trainer.train()
+
+model.save_pretrained_merged("outputs/lora-out-8b/merged", tokenizer, save_method = "merged_16bit",)
+
+api = HfApi()
+create_repo(f"jic062/{repo_name}", repo_type="model",private=True, token="")
+api.upload_folder(
+        folder_path="outputs/lora-out-8b/merged",
+        repo_id=f"jic062/{repo_name}",
+        repo_type="model",
+)
+wandb.finish()