Create README.md

README.md

@@ -0,0 +1,60 @@
+```
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = "unsloth/GLM-4.7-Flash",
+    max_seq_length = 2048, # Choose any for long context!
+    load_in_4bit = False,  # 4 bit quantization to reduce memory
+    load_in_8bit = False, # [NEW!] A bit more accurate, uses 2x memory
+    full_finetuning = False, # [NEW!] We have full finetuning now!
+    trust_remote_code = True,
+    unsloth_force_compile = False,
+)
+
+model = FastLanguageModel.get_peft_model(
+    model,
+    r = 8, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128
+    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
+                      "gate_proj", "up_proj", "down_proj",
+                      "in_proj", "out_proj",],
+    lora_alpha = 16,
+    lora_dropout = 0, # Supports any, but = 0 is optimized
+    bias = "none",    # Supports any, but = "none" is optimized
+    # [NEW] "unsloth" uses 30% less VRAM, fits 2x larger batch sizes!
+    use_gradient_checkpointing = "unsloth", # True or "unsloth" for very long context
+    random_state = 3407,
+    use_rslora = False,  # We support rank stabilized LoRA
+    loftq_config = None, # And LoftQ
+)
+
+dataset = load_dataset("unsloth/OpenMathReasoning-mini", split = "cot")
+
+# This step might take ~3m on this A100 notebook
+from trl import SFTTrainer, SFTConfig
+trainer = SFTTrainer(
+    model = model,
+    tokenizer = tokenizer,
+    train_dataset = dataset,
+    eval_dataset = None, # Can set up evaluation!
+    args = SFTConfig(
+        dataset_text_field = "text",
+        dataset_num_proc=1, # Increasing "might" throw error on Colab/other envs.
+        per_device_train_batch_size = 4,
+        gradient_accumulation_steps = 2, # Use GA to mimic batch size!
+        warmup_steps = 5,
+        # num_train_epochs = 1, # Set this for 1 full training run.
+        max_steps = 60,
+        learning_rate = 2e-4, # Reduce to 2e-5 for long training runs
+        logging_steps = 1,
+        optim = "adamw_8bit",
+        weight_decay = 0.001,
+        lr_scheduler_type = "linear",
+        seed = 3407,
+        report_to = "none", # Use TrackIO/WandB etc
+    ),
+)
+
+trainer = train_on_responses_only(
+    trainer,
+    instruction_part = "[gMASK]<sop><|user|>", # Updated for GLM
+    response_part = "<|assistant|><think>",
+)
+