update

examples/tutorials/by_deepspeed/ds_config/deepspeed_stage_3_config.json

@@ -0,0 +1,58 @@
+{
+  "gradient_accumulation_steps": "auto",
+  "gradient_clipping": "auto",
+  "steps_per_print": 200,
+  "train_batch_size": "auto",
+  "train_micro_batch_size_per_gpu": "auto",
+  "wall_clock_breakdown": false,
+  "optimizer": {
+    "type": "Adam",
+    "params": {
+      "lr": "auto",
+      "betas": "auto",
+      "eps": "auto",
+      "weight_decay": "auto"
+    }
+  },
+  "fp16": {
+    "enabled": "auto",
+    "loss_scale": 0,
+    "loss_scale_window": 1000,
+    "initial_scale_power": 16,
+    "hysteresis": 2,
+    "min_loss_scale": 1
+  },
+  "zero_optimization": {
+    "stage": 3,
+    "offload_optimizer": {
+      "device": "cpu",
+      "pin_memory": true
+    },
+    "offload_param": {
+      "device": "cpu",
+      "pin_memory": true
+    },
+    "overlap_comm": true,
+    "contiguous_gradients": true,
+    "sub_group_size": 1e9,
+    "reduce_bucket_size": "auto",
+    "stage3_prefetch_bucket_size": "auto",
+    "stage3_param_persistence_threshold": "auto",
+    "stage3_max_live_parameters": 1e9,
+    "stage3_max_reuse_distance": 1e9,
+    "stage3_gather_16bit_weights_on_model_save": true
+  },
+  "scheduler": {
+    "type": "WarmupLR",
+    "params": {
+      "warmup_min_lr": "auto",
+      "warmup_max_lr": "auto",
+      "warmup_num_steps": "auto"
+    }
+  },
+  "activation_checkpointing": {
+    "enabled": true,
+    "partition_activations": true,
+    "contiguous_memory_optimization": true
+  }
+}

examples/tutorials/by_deepspeed/requirements.txt

@@ -0,0 +1,4 @@
+datasets
+unsloth
+modelscope
+

examples/tutorials/by_deepspeed/step_2_train_model.py

@@ -0,0 +1,170 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+from pathlib import Path
+import platform
+
+# os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
+
+if platform.system() in ("Windows", "Darwin"):
+    from project_settings import project_path
+else:
+    project_path = os.path.abspath("../../../")
+    project_path = Path(project_path)
+
+from peft import LoraConfig
+# from transformers import AutoConfig, AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from modelscope import AutoConfig, AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
+from trl import SFTTrainer, SFTConfig
+from datasets import load_dataset
+import torch
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_name",
+        default="unsloth/Qwen3-8B",
+        type=str
+    )
+    parser.add_argument(
+        "--dataset_path",
+        default="miyuki2026/tutorials",
+        type=str
+    ),
+    parser.add_argument("--dataset_name", default=None, type=str),
+    parser.add_argument("--dataset_split", default=None, type=str),
+    parser.add_argument(
+        "--dataset_cache_dir",
+        default=(project_path / "hub_datasets").as_posix(),
+        type=str
+    ),
+    parser.add_argument("--dataset_streaming", default=None, type=str),
+    parser.add_argument("--valid_dataset_size", default=100, type=str),
+    parser.add_argument("--shuffle_buffer_size", default=5000, type=str),
+
+    parser.add_argument(
+        "--num_workers",
+        default=None if platform.system() == "Windows" else os.cpu_count() // 2,
+        type=str
+    ),
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    model = AutoModelForCausalLM.from_pretrained(
+        pretrained_model_name_or_path=args.model_name,
+        quantization_config=None,
+        # device_map="auto",
+        trust_remote_code=True
+    )
+    tokenizer = AutoTokenizer.from_pretrained(
+        pretrained_model_name_or_path=args.model_name,
+        trust_remote_code=True
+    )
+    print(model)
+
+    def format_func(example):
+        formated_text = tokenizer.apply_chat_template(
+            example["conversation"],
+            tokenize=False,  # 训练时部分词，true返回的是张量
+            add_generation_prompt=False,  # 训练期间要关闭，如果是推理则设为True
+        )
+        return {"formated_text": formated_text}
+
+    dataset_dict = load_dataset(
+        path=args.dataset_path,
+        name=args.dataset_name,
+        data_dir="keywords",
+        # data_dir="psychology",
+        split=args.dataset_split,
+        cache_dir=args.dataset_cache_dir,
+        # num_proc=args.num_workers if not args.dataset_streaming else None,
+        streaming=args.dataset_streaming,
+    )
+    dataset = dataset_dict["train"]
+    print(dataset)
+
+    if args.dataset_streaming:
+        valid_dataset = dataset.take(args.valid_dataset_size)
+        train_dataset = dataset.skip(args.valid_dataset_size)
+        train_dataset = train_dataset.shuffle(buffer_size=args.shuffle_buffer_size, seed=None)
+    else:
+        dataset = dataset.train_test_split(test_size=args.valid_dataset_size, seed=None)
+        train_dataset = dataset["train"]
+        valid_dataset = dataset["test"]
+
+    train_dataset = valid_dataset
+    train_dataset = train_dataset.map(
+        format_func,
+        batched=False,
+        remove_columns=train_dataset.column_names,
+    )
+
+    trainer = SFTTrainer(
+        model=model,
+        processing_class=tokenizer,  # 新写法
+        train_dataset=train_dataset,
+        eval_dataset=None,  # Can set up evaluation!
+        args=SFTConfig(
+            dataset_text_field="formated_text",
+            deepspeed="./ds_config/deepspeed_stage_3_config.json",  # 添加deepspeed配置文件
+            per_device_train_batch_size=1,
+            gradient_accumulation_steps=64,  # Use GA to mimic batch size!
+            warmup_steps=100,
+            num_train_epochs=1,  # Set this for 1 full training run.
+            # max_steps = 30,
+            learning_rate=3e-5,  # Reduce to 2e-5 for long training runs
+            logging_steps=1,
+            optim="adamw_8bit",
+            weight_decay=0,
+            lr_scheduler_type="constant_with_warmup",
+            seed=3407,
+            report_to="none",  # Use this for WandB etc
+        ),
+    )
+
+    # 显示当前内存统计信息
+    gpu_stats = torch.cuda.get_device_properties(0)
+    start_gpu_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
+    max_memory = round(gpu_stats.total_memory / 1024 / 1024 / 1024, 3)
+    print(f"GPU = {gpu_stats.name}. Max memory = {max_memory} GB.")
+    print(f"{start_gpu_memory} GB of memory reserved.")
+
+    trainer_stats = trainer.train()
+
+    # 显示最终内存和时间统计信息
+    used_memory = round(torch.cuda.max_memory_reserved() / 1024 / 1024 / 1024, 3)
+    used_memory_for_lora = round(used_memory - start_gpu_memory, 3)
+    used_percentage = round(used_memory / max_memory * 100, 3)
+    lora_percentage = round(used_memory_for_lora / max_memory * 100, 3)
+    print(f"{trainer_stats.metrics['train_runtime']} seconds used for training.")
+    print(
+        f"{round(trainer_stats.metrics['train_runtime'] / 60, 2)} minutes used for training."
+    )
+    print(f"Peak reserved memory = {used_memory} GB.")
+    print(f"Peak reserved memory for training = {used_memory_for_lora} GB.")
+    print(f"Peak reserved memory % of max memory = {used_percentage} %.")
+    print(f"Peak reserved memory for training % of max memory = {lora_percentage} %.")
+
+    # 只保存lora适配器参数
+    trained_models_dir = project_path / "trained_models" / "Qwen3-8B-sft-lora-adapter-transformers"
+    trained_models_dir.mkdir(parents=True, exist_ok=True)
+    trainer.model.save_pretrained(trained_models_dir.as_posix())
+    tokenizer.save_pretrained(trained_models_dir.as_posix())
+
+    # trained_models_dir = project_path / "trained_models" / "Qwen3-8B-sft-fp16"
+    # trained_models_dir.mkdir(parents=True, exist_ok=True)
+    # trainer.model.save_pretrained_merged(trained_models_dir.as_posix(), tokenizer, save_method="merged_16bit",)
+    # trained_models_dir = project_path / "trained_models" / "Qwen3-8B-sft-int4"
+    # trained_models_dir.mkdir(parents=True, exist_ok=True)
+    # trainer.model.save_pretrained_merged(trained_models_dir.as_posix(), tokenizer, save_method="merged_4bit",)
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/tutorials/lora_transformers/step_2_train_model.py

@@ -41,7 +41,7 @@ def get_args():
         type=str
     ),
     parser.add_argument("--dataset_streaming", default=None, type=str),
-    parser.add_argument("--valid_dataset_size", default=100, type=str),
+    parser.add_argument("--valid_dataset_size", default=1000, type=str),
     parser.add_argument("--shuffle_buffer_size", default=5000, type=str),
 
     parser.add_argument(
@@ -115,7 +115,7 @@ def main():
         train_dataset = dataset["train"]
         valid_dataset = dataset["test"]
 
-    train_dataset = valid_dataset
+    # train_dataset = valid_dataset
     train_dataset = train_dataset.map(
         format_func,
         batched=False,

examples/tutorials/lora_transformers/step_3_inter_model.py

@@ -69,7 +69,7 @@ def main():
 
     # 注入lora适配器
     model = PeftModel.from_pretrained(model, args.lora_adapter_path)
-    # model.merge_and_unload()
+    # model.merge_and_unload()  #这一步，真正将LoRA的AB矩阵融入进取。
     model.eval()
     # print(model)