update

examples/tutorials/dpo/ultrafeedback-dpo/{step_2_train_dpo_model_single_gpu.py → step_2_train_dpo_model_ddp_qlora.py}

@@ -3,11 +3,14 @@
 """
 https://huggingface.co/docs/trl/v0.16.1/en/sft_trainer
 
-单卡 V00 32G 全参微调
-python3 step_2_train_dpo_model_single_gpu.py
+多卡 V00 32G 全参微调
+python3 -m torch.distributed.run --nproc_per_node=4 step_2_train_dpo_model_ddp_qlora.py
+
+torchrun --nproc_per_node=4 step_2_train_dpo_model_ddp_qlora.py
 
 DPO本来就是风格微调，用LoRA 训练更合理，更科学。
 
+
 """
 import argparse
 import os
@@ -37,6 +40,8 @@ from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
 
 def get_args():
     parser = argparse.ArgumentParser()
+    parser.add_argument("--local_rank", type=int, default=0)  # torchrun会自动传递这个参数
+
     parser.add_argument(
         "--model_name",
         default=(project_path / "pretrained_models/jingyaogong/MiniMind2").as_posix() if debug_mode else "qgyd2021/Qwen2.5-0.5B-ultrachat-sft-deepspeed",
@@ -60,9 +65,12 @@ def get_args():
     ),
     parser.add_argument(
         "--output_model_dir",
-        default=(temp_directory / "trained_models/qwen2_5-0_5B-ultrafeedback-dpo-single-gpu").as_posix(),
+        default=(temp_directory / "trained_models/qwen2_5-0_5B-ultrafeedback-dpo-ddp-qlora").as_posix(),
         type=str
     ),
+
+    parser.add_argument("--beta", default=0.5, type=float),
+
     parser.add_argument(
         "--num_workers",
         default=None if debug_mode else os.cpu_count() // 2,
@@ -202,11 +210,14 @@ def main():
         optim="adamw_torch",
         report_to="none",
         max_length=1024 if debug_mode else 2048,  # prompt + chosen 的最大长度
-        max_prompt_length=512 if debug_mode else 1024,  # prompt 的最大长度
         # DPO 特定参数
-        beta=0.1,  # DPO 的温度参数，控制对 preference 的置信度
+        beta=args.beta,  # DPO 的温度参数，控制对 preference 的置信度
         remove_unused_columns=False,
         dataloader_pin_memory=False,
+
+        # ddp_find_unused_parameters=False,  # 告诉DDP忽略未使用的参数
+        local_rank=args.local_rank,  # 传递当前进程的local_rank
+
     )
 
     trainer = DPOTrainer(
@@ -222,9 +233,10 @@ def main():
     trainer.train()
 
     # 保存模型
-    print(f"保存模型到: {args.output_model_dir}")
-    trainer.save_model()
-    tokenizer.save_pretrained(args.output_model_dir)
+    if args.local_rank == 0:  # 只在主进程保存
+        print(f"保存模型到: {args.output_model_dir}")
+        trainer.save_model()
+        tokenizer.save_pretrained(args.output_model_dir)
 
     print("DPO 训练完成!")
     return

examples/tutorials/dpo/ultrafeedback-dpo/step_2_train_dpo_model_single_gpu_qlora.py

@@ -0,0 +1,249 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+https://huggingface.co/docs/trl/v0.16.1/en/sft_trainer
+
+单卡 V00 32G 全参微调
+python3 step_2_train_dpo_model_single_gpu.py
+
+DPO本来就是风格微调，用LoRA 训练更合理，更科学。
+
+
+---------------
+{'loss': '0.6324', 'grad_norm': '1.082', 'learning_rate': '3.257e-06', 'rewards/chosen': '0.2385', 'rewards/rejected': '-0.2982', 'rewards/accuracies': '0.6438', 'rewards/margins': '0.5366', 'logps/chosen': '-367', 'logps/rejected': '-336.3', 'logits/chosen': '-1.805', 'logits/rejected': '-1.832', 'epoch': '0.7433'}
+logps/chosen 比 logps/rejected 小，说明模型生成优选项的概率小于拒选项。
+最终的模型应是 logps/chosen 更大。
+如果loss损失不直降，就调大 LoRA的 rank。
+
+---------------
+此模型训练一开始就倾向于生成拒选项，0.74epoch时仍然倾向于生成拒选项。
+GPT建议调大 beta，限制当前模型的自由度。
+当前 beta=0.1 改为 0.5
+
+
+"""
+import argparse
+import os
+from pathlib import Path
+import platform
+
+# os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
+
+debug_mode = True if platform.system() in ("Windows", "Darwin") else False
+print(f"debug_mode: {debug_mode}")
+
+if platform.system() in ("Windows", "Darwin"):
+    from project_settings import project_path, temp_directory
+else:
+    project_path = os.path.abspath("../../../")
+    project_path = Path(project_path)
+    temp_directory = Path("/root/autodl-tmp/OpenMiniMind/temp")
+
+from datasets import load_dataset
+import torch
+
+from modelscope import AutoModelForCausalLM, AutoTokenizer
+from transformers import BitsAndBytesConfig
+from trl import DPOConfig, DPOTrainer
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_name",
+        default=(project_path / "pretrained_models/jingyaogong/MiniMind2").as_posix() if debug_mode else "qgyd2021/Qwen2.5-0.5B-ultrachat-sft-deepspeed",
+        type=str
+    ),
+    parser.add_argument(
+        "--dataset_path",
+        default="HuggingFaceH4/ultrafeedback_binarized",
+        # default="miyuki2026/tutorials" if debug_mode else "HuggingFaceH4/ultrachat_200k",
+        type=str
+    ),
+    parser.add_argument(
+        "--dataset_cache_dir",
+        default=(temp_directory / "hub_datasets").as_posix(),
+        type=str
+    ),
+    parser.add_argument(
+        "--model_cache_dir",
+        default=(temp_directory / "hub_models").as_posix(),
+        type=str
+    ),
+    parser.add_argument(
+        "--output_model_dir",
+        default=(temp_directory / "trained_models/qwen2_5-0_5B-ultrafeedback-dpo-single-gpu-qlora").as_posix(),
+        type=str
+    ),
+
+    parser.add_argument("--beta", default=0.5, type=float),
+
+    parser.add_argument(
+        "--num_workers",
+        default=None if debug_mode else os.cpu_count() // 2,
+        type=int
+    ),
+    args = parser.parse_args()
+    return args
+
+
+def format_func(examples, tokenizer):
+    chosen = examples["chosen"]
+    rejected = examples["rejected"]
+
+    chosen_prompt = chosen[:-1]
+    chosen_response = chosen[-1]
+
+    rejected_prompt = rejected[:-1]
+    rejected_response = rejected[-1]
+
+    chosen_prompt_text = tokenizer.apply_chat_template(
+        conversation=chosen_prompt,
+        tokenize=False,
+        add_generation_prompt=True,  # DPO 需要添加生成提示，让模型知道要从这里开始生成
+    )
+    rejected_prompt_text = tokenizer.apply_chat_template(
+        conversation=rejected_prompt,
+        tokenize=False,
+        add_generation_prompt=True,  # DPO 需要添加生成提示，让模型知道要从这里开始生成
+    )
+    if chosen_prompt_text != rejected_prompt_text:
+        raise AssertionError()
+
+    chosen_response_role = chosen_response["role"]
+    chosen_response_text = chosen_response["content"]
+    if chosen_response_role != "assistant":
+        raise AssertionError()
+
+    rejected_response_role = rejected_response["role"]
+    rejected_response_text = rejected_response["content"]
+    if rejected_response_role != "assistant":
+        raise AssertionError()
+
+    result = {
+        "prompt": chosen_prompt_text,
+        "chosen": chosen_response_text,
+        "rejected": rejected_response_text,
+    }
+    return result
+
+
+def main():
+    args = get_args()
+
+    os.environ["MODELSCOPE_CACHE"] = args.model_cache_dir
+
+    bnb_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_dtype=torch.float16,
+        bnb_4bit_use_double_quant=True,
+        bnb_4bit_quant_storage=torch.uint8,
+    )
+
+    model = AutoModelForCausalLM.from_pretrained(
+        args.model_name,
+        cache_dir=args.model_cache_dir,
+        quantization_config=bnb_config,
+        device_map="auto",
+        trust_remote_code=True,
+        use_cache=False,  # 训练时禁用KV cache
+    )
+    ref_model = AutoModelForCausalLM.from_pretrained(
+        args.model_name,
+        cache_dir=args.model_cache_dir,
+        trust_remote_code=True,
+        quantization_config=bnb_config,
+        device_map="auto",
+        use_cache=False,
+    )
+    model = prepare_model_for_kbit_training(model)
+    ref_model = prepare_model_for_kbit_training(ref_model)
+
+    lora_config = LoraConfig(
+        r=16,
+        lora_alpha=32,
+        target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+        lora_dropout=0.1,
+        bias="none",
+        task_type="CAUSAL_LM",
+    )
+    model = get_peft_model(model, lora_config)
+    ref_model = get_peft_model(ref_model, lora_config)
+    model.print_trainable_parameters()
+
+    tokenizer = AutoTokenizer.from_pretrained(
+        args.model_name,
+        cache_dir=args.model_cache_dir,
+        trust_remote_code=True,
+        padding_side="left",  # DPO需要left padding
+    )
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+        tokenizer.pad_token_id = tokenizer.eos_token_id
+
+    print(model)
+    print(ref_model)
+    print(tokenizer)
+
+    dataset_dict = load_dataset(
+        path=args.dataset_path,
+        cache_dir=args.dataset_cache_dir,
+    )
+    train_dataset = dataset_dict["train_prefs"]
+    # test_dataset = dataset_dict["test_prefs"]
+
+    train_dataset = train_dataset.map(
+        lambda x: format_func(x, tokenizer),
+        batched=False,
+        num_proc=args.num_workers,
+        remove_columns=train_dataset.column_names,
+    )
+
+    dpo_config = DPOConfig(
+        output_dir=args.output_model_dir,
+        num_train_epochs=1,
+        per_device_train_batch_size=1 if debug_mode else 2,
+        gradient_accumulation_steps=1 if debug_mode else 8,
+        save_strategy="steps",
+        save_steps=100,
+        save_total_limit=2,
+        logging_steps=10,
+        learning_rate=2e-5,
+        warmup_steps=100,
+        lr_scheduler_type="cosine",
+        fp16=True,
+        gradient_checkpointing=True,  # 如果内存紧张，可以设为 True
+        optim="adamw_torch",
+        report_to="none",
+        max_length=1024 if debug_mode else 2048,  # prompt + chosen 的最大长度
+        # DPO 特定参数
+        beta=args.beta,  # DPO 的温度参数，控制对 preference 的置信度
+        remove_unused_columns=False,
+        dataloader_pin_memory=False,
+    )
+
+    trainer = DPOTrainer(
+        model=model,
+        ref_model=ref_model,
+        args=dpo_config,
+        train_dataset=train_dataset,
+        # DPOTrainer 会自动处理数据，不需要 data_collator
+    )
+
+    # 开始训练
+    print("开始 DPO 训练...")
+    trainer.train()
+
+    # 保存模型
+    print(f"保存模型到: {args.output_model_dir}")
+    trainer.save_model()
+    tokenizer.save_pretrained(args.output_model_dir)
+
+    print("DPO 训练完成!")
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/tutorials/grpo/step_2_train_grpo_model.py

@@ -0,0 +1,11 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+
+"""
+
+
+
+
+if __name__ == "__main__":
+    pass