update

examples/tutorials/dpo/ultrafeedback-dpo-unsloth/requirements.txt

@@ -0,0 +1,8 @@
+transformers
+peft
+torch
+modelscope
+datasets
+trl
+bitsandbytes
+unsloth

examples/tutorials/dpo/{ultrafeedback-dpo → ultrafeedback-dpo-unsloth}/step_2_train_dpo_model_unsloth_ddp_qlora.py

@@ -15,7 +15,7 @@ DPO本来就是风格微调，用LoRA 训练更合理，更科学。
 nohup torchrun --nproc_per_node=4 step_2_train_dpo_model_unsloth_ddp_qlora.py \
 --num_train_epochs 5 \
 --learning_rate 5e-5 \
---dpo_beta 0.05 \
+--dpo_beta 0.1 \
 --lora_rank 32 \
 &
 
@@ -217,6 +217,9 @@ def main():
         num_proc=args.num_workers,
         remove_columns=valid_dataset.column_names,
     )
+    if is_main_process:
+        print(f"train_dataset mapped count: {len(train_dataset)}")
+        print(f"valid_dataset mapped count: {len(valid_dataset)}")
 
     def filter_long_samples(example):
         # 简单估计长度，实际训练时会由tokenizer处理
@@ -230,6 +233,9 @@ def main():
         return True
     train_dataset = train_dataset.filter(filter_long_samples)
     valid_dataset = valid_dataset.filter(filter_long_samples)
+    if is_main_process:
+        print(f"train_dataset filtered count: {len(train_dataset)}")
+        print(f"valid_dataset filtered count: {len(valid_dataset)}")
 
     # 配置 DPO 训练参数
     dpo_config = DPOConfig(
@@ -253,13 +259,16 @@ def main():
         bf16=is_bfloat16_supported(),  # 如果支持bfloat16则使用bf16
         optim="adamw_8bit",  # 使用8bit优化器节省显存
         report_to="none",
-        max_length=args.max_seq_length,  # prompt + chosen 的最大长度
-        max_prompt_length=args.max_seq_length // 2,  # prompt 的最大长度
+        load_best_model_at_end=True,  # 训练结束时加载最佳模型
+        metric_for_best_model="eval_rewards/margins",
+        greater_is_better=True,  # margin 越大越好
 
         # DPO 特定参数
         beta=args.dpo_beta,  # DPO 的温度参数
         remove_unused_columns=False,
         dataloader_pin_memory=False,
+        max_prompt_length=args.max_seq_length // 2,  # prompt 的最大长度
+        max_length=args.max_seq_length,  # prompt + chosen 的最大长度
 
         # DDP 相关参数
         ddp_find_unused_parameters=False,  # 重要：告诉DDP忽略未使用的参数

examples/tutorials/dpo/ultrafeedback-dpo-unsloth/step_3_infer.py

@@ -0,0 +1,107 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+import argparse
+import os
+from pathlib import Path
+import platform
+
+# os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
+os.environ["UNSLOTH_USE_MODELSCOPE"] = "1"
+
+if platform.system() in ("Windows", "Darwin"):
+    from project_settings import project_path, temp_directory
+else:
+    project_path = os.path.abspath("../../../")
+    project_path = Path(project_path)
+    temp_directory = Path("/root/autodl-tmp/OpenMiniMind/temp")
+
+from unsloth import FastLanguageModel
+from transformers import TextStreamer
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--model_name",
+        default="qgyd2021/Qwen2.5-0.5B-ultrachat-sft-deepspeed",
+        type=str
+    ),
+    parser.add_argument(
+        "--model_cache_dir",
+        default=(temp_directory / "hub_models").as_posix(),
+        type=str
+    ),
+    parser.add_argument(
+        "--lora_adapter_path",
+        default=(temp_directory / "trained_models/qwen2_5-0_5B-ultrafeedback-dpo-ddp-qlora/checkpoint-800").as_posix(),
+        type=str
+    ),
+    parser.add_argument(
+        "--max_new_tokens",
+        default=1024, # 8192, 128
+        type=int, help="最大生成长度（注意：并非模型实际长文本能力）"
+    ),
+    parser.add_argument("--top_p", default=0.85, type=float, help="nucleus采样阈值（0-1）"),
+    parser.add_argument("--temperature", default=0.85, type=float, help="生成温度，控制随机性（0-1，越大越随机）"),
+
+    parser.add_argument(
+        "--num_workers",
+        default=None if platform.system() == "Windows" else os.cpu_count() // 2,
+        type=str
+    ),
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = get_args()
+
+    os.environ["MODELSCOPE_CACHE"] = args.model_cache_dir
+
+    model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name=args.model_name,
+        max_seq_length=2048,  # 支持32K+长上下文
+        device_map="auto",
+        dtype=None,  # 自动选择最优精度
+        load_in_4bit=True,  # 4bit量化节省70%显存
+        cache_dir=args.model_cache_dir,
+    )
+
+    # 2、注入lora适配器
+    model.load_adapter(args.lora_adapter_path)
+
+    # 启用unsloth推理加速
+    FastLanguageModel.for_inference(model)
+    model.eval()
+
+    messages = [
+        {
+            "role": "user",
+            "content": "how can i develop a habit of drawing daily"
+        }
+    ]
+    format_messages = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,  # 训练时部分词，true返回的是张量
+        add_generation_prompt=True,  # 训练期间要关闭，如果是推理则设为True
+    )
+
+    # 4、调用tokenizer得到input
+    inputs = tokenizer(format_messages, return_tensors="pt").to(model.device)
+
+    # 5、调用model.generate()
+    generated_ids = model.generate(
+        **inputs,
+        max_new_tokens=args.max_new_tokens, do_sample=True,
+        streamer=TextStreamer(tokenizer, skip_prompt=True, skip_special_tokens=True),
+        pad_token_id=tokenizer.pad_token_id, eos_token_id=tokenizer.eos_token_id,
+        top_p=args.top_p, temperature=args.temperature, repetition_penalty=1.0,
+    )
+
+    response = tokenizer.decode(generated_ids[0][len(inputs["input_ids"][0]):], skip_special_tokens=True)
+    print(f"response: {response}")
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/tutorials/dpo/ultrafeedback-dpo-unsloth/step_3_merge.py

@@ -0,0 +1,45 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+from unsloth import FastLanguageModel
+import torch
+
+# 1. 加载原始模型（必须与训练时完全一致）
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name="qgyd2021/Qwen2.5-0.5B-ultrachat-sft-deepspeed",  # 你的基础模型
+    max_seq_length=2048,
+    dtype=None,
+    load_in_4bit=True,  # 加载为4bit以节省内存
+)
+
+# 2. 加载训练好的 LoRA 权重
+model = FastLanguageModel.get_peft_model(
+    model,
+    r=32,  # 必须与训练时的 lora_rank 一致
+    lora_alpha=64,  # lora_rank * 2
+    target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+    lora_dropout=0,
+    bias="none",
+    use_gradient_checkpointing="unsloth",
+    max_seq_length=2048,
+)
+
+# 加载训练好的 adapter 权重
+model.load_adapter("你的adapter目录")  # 替换为你的目录路径
+
+# 3. 合并并保存为16位精度（推荐用于上传）
+model.save_pretrained_merged(
+    "merged_model_16bit",  # 输出目录
+    tokenizer,
+    save_method="merged_16bit",  # 合并为16位
+)
+
+# 或者合并为4位量化（更小，但可能影响精度）
+model.save_pretrained_merged(
+    "merged_model_4bit",
+    tokenizer,
+    save_method="merged_4bit",
+)
+
+
+if __name__ == "__main__":
+    pass