Update README.md

README.md

@@ -6,7 +6,50 @@ tags:
 
 # Model Card for Model ID
 
-<!-- Provide a quick summary of what the model is/does. -->
+model_name =   "unsloth/llama-3-8b-Instruct-bnb-4bit",
+
+
+model = FastLanguageModel.get_peft_model(
+    model,
+    r = 64, 
+    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
+                      "gate_proj", "up_proj", "down_proj",],
+    lora_alpha = 16,
+    lora_dropout = 0.2, # Supports any, but = 0 is optimized
+    bias = "none",    # Supports any, but = "none" is optimized
+    # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
+    use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
+    random_state = 3407,
+    use_rslora = False,  # We support rank stabilized LoRA
+    loftq_config = None, # And LoftQ
+)
+
+trainer = SFTTrainer(
+    model=model,
+    tokenizer=tokenizer,
+    train_dataset=dataset_transformed,
+    dataset_text_field="text",
+    max_seq_length=max_seq_length,
+    dataset_num_proc=2,
+    packing=False,
+    args=TrainingArguments(
+        per_device_train_batch_size=2,
+        gradient_accumulation_steps=4,
+        warmup_steps=5,
+        num_train_epochs=2,
+        learning_rate=2e-4,
+        fp16=not is_bfloat16_supported(),
+        bf16=is_bfloat16_supported(),
+        logging_steps=1,
+        optim="adamw_8bit",
+        weight_decay=0.01,
+        lr_scheduler_type="linear",
+        seed=3407,
+        output_dir="outputs",
+        max_grad_norm=1.0,  # Added for gradient clipping
+    ),
+)
+trainer_stats = trainer.train()
 
 
 
@@ -40,6 +83,8 @@ This is the model card of a 🤗 transformers model that has been pushed on the
 
 ### Direct Use
 
+
+
 <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
 
 [More Information Needed]