update

examples/tutorials/lora_transformers/requirements.txt

@@ -0,0 +1,4 @@
+datasets
+unsloth
+modelscope
+

examples/tutorials/lora_transformers/step_2_train_model.py

@@ -0,0 +1,188 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+from pathlib import Path
+import platform
+
+# os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
+
+if platform.system() in ("Windows", "Darwin"):
+    from project_settings import project_path
+else:
+    project_path = os.path.abspath("../../../")
+    project_path = Path(project_path)
+
+from peft import LoraConfig
+# from transformers import AutoConfig, AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from modelscope import AutoConfig, AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from trl import SFTTrainer, SFTConfig
+from datasets import load_dataset
+import torch
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_name",
+        default="unsloth/Qwen3-8B-unsloth-bnb-4bit",
+        type=str
+    )
+    parser.add_argument(
+        "--dataset_path",
+        default="miyuki2026/tutorials",
+        type=str
+    ),
+    parser.add_argument("--dataset_name", default=None, type=str),
+    parser.add_argument("--dataset_split", default=None, type=str),
+    parser.add_argument(
+        "--dataset_cache_dir",
+        default=(project_path / "hub_datasets").as_posix(),
+        type=str
+    ),
+    parser.add_argument("--dataset_streaming", default=None, type=str),
+    parser.add_argument("--valid_dataset_size", default=1000, type=str),
+    parser.add_argument("--shuffle_buffer_size", default=5000, type=str),
+
+    parser.add_argument(
+        "--num_workers",
+        default=None if platform.system() == "Windows" else os.cpu_count() // 2,
+        type=str
+    ),
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    bnb_config = BitsAndBytesConfig(
+        load_in_4bit=True,  # 启用4-bit量化
+        bnb_4bit_quant_type="nf4",  # 量化类型
+        bnb_4bit_compute_dtype=torch.float16,
+        bnb_4bit_use_double_quant=True  # 嵌套量化节省更多内存
+    )
+    model = AutoModelForCausalLM.from_pretrained(
+        pretrained_model_name_or_path=args.model_name,
+        quantization_config=bnb_config,
+        device_map="auto",
+        trust_remote_code=True
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        pretrained_model_name_or_path=args.model_name,
+        trust_remote_code=True
+    )
+    peft_config = LoraConfig(
+        r=32,  # LoRA秩
+        lora_alpha=32,  # 缩放因子
+        target_modules=[
+            "q_proj", "k_proj", "v_proj", "o_proj",
+            "gate_proj", "up_proj", "down_proj"
+        ],
+        lora_dropout=0.,  # Dropout率
+        bias="none",  # 偏置处理方式
+        task_type="CAUSAL_LM"  # 任务类型
+    )
+    print(model)
+
+    def format_func(example):
+        formated_text = tokenizer.apply_chat_template(
+            example["conversation"],
+            tokenize=False,  # 训练时部分词，true返回的是张量
+            add_generation_prompt=False,  # 训练期间要关闭，如果是推理则设为True
+        )
+        return {"formated_text": formated_text}
+
+    dataset_dict = load_dataset(
+        path=args.dataset_path,
+        name=args.dataset_name,
+        data_dir="keywords",
+        # data_dir="psychology",
+        split=args.dataset_split,
+        cache_dir=args.dataset_cache_dir,
+        # num_proc=args.num_workers if not args.dataset_streaming else None,
+        streaming=args.dataset_streaming,
+    )
+    dataset = dataset_dict["train"]
+
+    if args.dataset_streaming:
+        valid_dataset = dataset.take(args.valid_dataset_size)
+        train_dataset = dataset.skip(args.valid_dataset_size)
+        train_dataset = train_dataset.shuffle(buffer_size=args.shuffle_buffer_size, seed=None)
+    else:
+        dataset = dataset.train_test_split(test_size=args.valid_dataset_size, seed=None)
+        train_dataset = dataset["train"]
+        valid_dataset = dataset["test"]
+
+    train_dataset = train_dataset.map(
+        format_func,
+        batched=False,
+        remove_columns=train_dataset.column_names,
+    )
+    print(train_dataset)
+
+    trainer = SFTTrainer(
+        model=model,
+        processing_class=tokenizer,
+        # tokenizer=tokenizer,
+        peft_config=peft_config,
+        train_dataset=train_dataset,
+        eval_dataset=None,  # Can set up evaluation!
+        args=SFTConfig(
+            dataset_text_field="text",
+            per_device_train_batch_size=1,
+            gradient_accumulation_steps=2,  # Use GA to mimic batch size!
+            warmup_steps=5,
+            num_train_epochs=1,  # Set this for 1 full training run.
+            # max_steps = 30,
+            learning_rate=2e-4,  # Reduce to 2e-5 for long training runs
+            logging_steps=1,
+            optim="adamw_8bit",
+            weight_decay=0.01,
+            lr_scheduler_type="linear",
+            seed=3407,
+            report_to="none",  # Use this for WandB etc
+        ),
+    )
+
+    # 显示当前内存统计信息
+    gpu_stats = torch.cuda.get_device_properties(0)
+    start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
+    max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
+    print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
+    print(f"{start_gpu_memory} GB of memory reserved.")
+
+    trainer_stats = trainer.train()
+
+    # 显示最终内存和时间统计信息
+    used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
+    used_memory_for_lora = round(used_memory - start_gpu_memory, 3)
+    used_percentage = round(used_memory / max_memory * 100, 3)
+    lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)
+    print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.")
+    print(
+        f"{round(trainer_stats.metrics['train_runtime'] / 60, 2)} minutes used for training."
+    )
+    print(f"Peak reserved memory = {used_memory} GB.")
+    print(f"Peak reserved memory for training = {used_memory_for_lora} GB.")
+    print(f"Peak reserved memory % of max memory = {used_percentage} %.")
+    print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.")
+
+    # ==================== 4.保存训练结果 ====================================
+    # 只保存lora适配器参数
+    trained_models_dir = project_path / "trained_models" / "Qwen3-8B-sft-lora-adapter-unsloth"
+    trained_models_dir.mkdir(parents=True, exist_ok=True)
+    model.save_pretrained(trained_models_dir.as_posix())
+    tokenizer.save_pretrained(trained_models_dir.as_posix())
+
+    # trained_models_dir = project_path / "trained_models" / "Qwen3-8B-sft-fp16"
+    # trained_models_dir.mkdir(parents=True, exist_ok=True)
+    # model.save_pretrained_merged(trained_models_dir.as_posix(), tokenizer, save_method="merged_16bit",)
+    # trained_models_dir = project_path / "trained_models" / "Qwen3-8B-sft-int4"
+    # trained_models_dir.mkdir(parents=True, exist_ok=True)
+    # model.save_pretrained_merged(trained_models_dir.as_posix(), tokenizer, save_method="merged_4bit",)
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/tutorials/lora_unsloth/step_2_train_model.py

@@ -40,9 +40,8 @@ def get_args():
         type=str
     ),
     parser.add_argument("--dataset_streaming", default=None, type=str),
-
-    parser.add_argument("--valid_dataset_size", default=None, type=str),
-    parser.add_argument("--shuffle_buffer_size", default=None, type=str),
+    parser.add_argument("--valid_dataset_size", default=1000, type=str),
+    parser.add_argument("--shuffle_buffer_size", default=5000, type=str),
 
     parser.add_argument(
         "--num_workers",

examples/tutorials/lora_unsloth/step_4_evaluation.py

@@ -46,8 +46,8 @@ def get_args():
         type=str
     ),
     parser.add_argument("--dataset_streaming", default=None, type=str),
-    parser.add_argument("--valid_dataset_size", default=None, type=str),
-    parser.add_argument("--shuffle_buffer_size", default=None, type=str),
+    parser.add_argument("--valid_dataset_size", default=1000, type=str),
+    parser.add_argument("--shuffle_buffer_size", default=5000, type=str),
 
     parser.add_argument(
         "--max_new_tokens",