update

examples/tutorials/dpo/ultrafeedback-dpo/step_2_train_dpo_model_ddp_qlora.py

@@ -6,7 +6,7 @@ https://huggingface.co/docs/trl/v0.16.1/en/sft_trainer
 多卡 V00 32G 全参微调
 python3 -m torch.distributed.run --nproc_per_node=2 step_2_train_dpo_model_ddp_qlora.py
 
-torchrun --nproc_per_node=2 step_2_train_dpo_model_ddp_qlora.py
+torchrun --nproc_per_node=4 step_2_train_dpo_model_ddp_qlora.py
 
 DPO本来就是风格微调，用LoRA 训练更合理，更科学。
 
@@ -14,7 +14,8 @@ DPO本来就是风格微调，用LoRA 训练更合理，更科学。
 ----------
 
 nohup torchrun --nproc_per_node=2 step_2_train_dpo_model_ddp_qlora.py \
---learning_rate 5e-5
+--num_train_epochs 5 \
+--learning_rate 5e-5 \
 --dpo_beta 0.05 \
 --lora_rank 32 \
 &
@@ -80,6 +81,7 @@ def get_args():
         type=str
     ),
 
+    parser.add_argument("--num_train_epochs", default=3, type=int),
     parser.add_argument("--learning_rate", default=2e-5, type=float),
     parser.add_argument("--dpo_beta", default=0.5, type=float),
     parser.add_argument("--lora_rank", default=32, type=int),
@@ -221,10 +223,13 @@ def main():
         num_proc=args.num_workers,
         remove_columns=train_dataset.column_names,
     )
+    train_dataset = train_dataset.filter(
+        function=lambda x: len(x["input_ids"]) < 2048
+    )
 
     dpo_config = DPOConfig(
         output_dir=args.output_model_dir,
-        num_train_epochs=1,
+        num_train_epochs=args.num_train_epochs,
         per_device_train_batch_size=1 if debug_mode else 2,
         gradient_accumulation_steps=1 if debug_mode else 8,
         save_strategy="steps",

examples/tutorials/dpo/ultrafeedback-dpo/step_2_train_dpo_model_unsloth_ddp_qlora.py

@@ -0,0 +1,301 @@
+#!/usr/bin/python3
+# -*- coding: utf-8 -*-
+"""
+https://huggingface.co/docs/trl/v0.16.1/en/sft_trainer
+
+多卡 V00 32G 全参微调
+python3 -m torch.distributed.run --nproc_per_node=4 step_2_train_dpo_model_unsloth_ddp_qlora.py
+
+torchrun --nproc_per_node=4 step_2_train_dpo_model_unsloth_ddp_qlora.py
+
+DPO本来就是风格微调，用LoRA 训练更合理，更科学。
+
+----------
+
+nohup torchrun --nproc_per_node=4 step_2_train_dpo_model_unsloth_ddp_qlora.py \
+--num_train_epochs 5 \
+--learning_rate 5e-5 \
+--dpo_beta 0.05 \
+--lora_rank 32 \
+&
+
+kill -9 `ps -aef | grep 'step_2_train_dpo_model_unsloth_ddp_qlora.py' | grep -v grep | awk '{print $2}'`
+
+
+"""
+import argparse
+import os
+from pathlib import Path
+import platform
+
+# os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"
+os.environ["UNSLOTH_USE_MODELSCOPE"] = "1"
+
+debug_mode = True if platform.system() in ("Windows", "Darwin") else False
+print(f"debug_mode: {debug_mode}")
+
+if platform.system() in ("Windows", "Darwin"):
+    from project_settings import project_path, temp_directory
+else:
+    project_path = os.path.abspath("../../../")
+    project_path = Path(project_path)
+    temp_directory = Path("/root/autodl-tmp/OpenMiniMind/temp")
+
+from datasets import load_dataset
+
+from trl import DPOConfig, DPOTrainer
+from unsloth import FastLanguageModel, is_bfloat16_supported
+
+
+def get_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--local_rank", type=int, default=-1)  # torchrun会自动传递这个参数
+
+    parser.add_argument(
+        "--model_name",
+        default=(project_path / "pretrained_models/jingyaogong/MiniMind2").as_posix() if debug_mode else "qgyd2021/Qwen2.5-0.5B-ultrachat-sft-deepspeed",
+        type=str
+    ),
+    parser.add_argument(
+        "--dataset_path",
+        default="HuggingFaceH4/ultrafeedback_binarized",
+        # default="miyuki2026/tutorials" if debug_mode else "HuggingFaceH4/ultrachat_200k",
+        type=str
+    ),
+    parser.add_argument(
+        "--dataset_cache_dir",
+        default=(temp_directory / "hub_datasets").as_posix(),
+        type=str
+    ),
+    parser.add_argument(
+        "--model_cache_dir",
+        default=(temp_directory / "hub_models").as_posix(),
+        type=str
+    ),
+    parser.add_argument(
+        "--output_model_dir",
+        default=(temp_directory / "trained_models/qwen2_5-0_5B-ultrafeedback-dpo-ddp-qlora").as_posix(),
+        type=str
+    ),
+
+    parser.add_argument("--max_seq_length",
+                        default=1024 if debug_mode else 2048,
+                        type=int
+                        )
+
+    parser.add_argument("--num_train_epochs", default=3, type=int),
+    parser.add_argument("--learning_rate", default=2e-5, type=float),
+    parser.add_argument("--dpo_beta", default=0.5, type=float),
+    parser.add_argument("--lora_rank", default=32, type=int),
+
+    parser.add_argument(
+        "--num_workers",
+        default=None if debug_mode else os.cpu_count() // 2,
+        type=int
+    ),
+    args = parser.parse_args()
+    return args
+
+
+def format_func(examples, tokenizer):
+    chosen = examples["chosen"]
+    rejected = examples["rejected"]
+
+    chosen_prompt = chosen[:-1]
+    chosen_response = chosen[-1]
+
+    rejected_prompt = rejected[:-1]
+    rejected_response = rejected[-1]
+
+    chosen_prompt_text = tokenizer.apply_chat_template(
+        conversation=chosen_prompt,
+        tokenize=False,
+        add_generation_prompt=True,  # DPO 需要添加生成提示，让模型知道要从这里开始生成
+    )
+    rejected_prompt_text = tokenizer.apply_chat_template(
+        conversation=rejected_prompt,
+        tokenize=False,
+        add_generation_prompt=True,  # DPO 需要添加生成提示，让模型知道要从这里开始生成
+    )
+    if chosen_prompt_text != rejected_prompt_text:
+        raise AssertionError()
+
+    chosen_response_role = chosen_response["role"]
+    chosen_response_text = chosen_response["content"]
+    if chosen_response_role != "assistant":
+        raise AssertionError()
+
+    rejected_response_role = rejected_response["role"]
+    rejected_response_text = rejected_response["content"]
+    if rejected_response_role != "assistant":
+        raise AssertionError()
+
+    result = {
+        "prompt": chosen_prompt_text,
+        "chosen": chosen_response_text,
+        "rejected": rejected_response_text,
+    }
+    return result
+
+
+def main():
+    args = get_args()
+
+    if args.local_rank == -1:
+        local_rank = os.environ.get("LOCAL_RANK", 0)
+        local_rank = int(local_rank)
+    else:
+        local_rank = args.local_rank
+    is_main_process = True if local_rank in (0, -1) else False
+
+    # 设置缓存目录
+    os.environ["MODELSCOPE_CACHE"] = args.model_cache_dir
+    os.environ["HF_DATASETS_CACHE"] = args.dataset_cache_dir
+
+    model, tokenizer = FastLanguageModel.from_pretrained(
+        model_name=args.model_name,
+        max_seq_length=args.max_seq_length,
+        dtype=None,
+        load_in_4bit=True,
+        cache_dir=args.model_cache_dir,
+    )
+    model = FastLanguageModel.get_peft_model(
+        model,
+        r=args.lora_rank,
+        lora_alpha=args.lora_rank * 2,
+        target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+        lora_dropout=0,  # Unsloth 推荐使用 0
+        bias="none",  # Unsloth 推荐使用 "none"
+        use_gradient_checkpointing="unsloth",  # 使用 unsloth 的梯度检查点实现
+        random_state=42,
+        max_seq_length=args.max_seq_length,  # 重要：设置最大序列长度
+    )
+
+    # 创建参考模型（使用相同的权重，但不训练）
+    # 在 unsloth 中，我们可以直接使用原始模型作为参考模型
+    # 但需要确保它也是量化版本
+    ref_model, _ = FastLanguageModel.from_pretrained(
+        model_name=args.model_name,
+        max_seq_length=args.max_seq_length,
+        dtype=None,
+        load_in_4bit=True,
+        cache_dir=args.model_cache_dir,
+    )
+    ref_model.eval()
+
+    if is_main_process:
+        model.print_trainable_parameters()
+        print(f"model device: {model.device}")
+        print(f"ref_model device: {ref_model.device}")
+
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+        tokenizer.pad_token_id = tokenizer.eos_token_id
+    tokenizer.padding_side = "left"  # DPO需要left padding
+
+    print(model)
+    print(ref_model)
+    print(tokenizer)
+
+    dataset_dict = load_dataset(
+        path=args.dataset_path,
+        cache_dir=args.dataset_cache_dir,
+    )
+    train_dataset = dataset_dict["train_prefs"]
+    valid_dataset = dataset_dict["test_prefs"]
+
+    train_dataset = train_dataset.map(
+        lambda x: format_func(x, tokenizer),
+        batched=False,
+        num_proc=args.num_workers,
+        remove_columns=train_dataset.column_names,
+    )
+    valid_dataset = valid_dataset.map(
+        lambda x: format_func(x, tokenizer),
+        batched=False,
+        num_proc=args.num_workers,
+        remove_columns=valid_dataset.column_names,
+    )
+
+    def filter_long_samples(example):
+        # 简单估计长度，实际训练时会由tokenizer处理
+        prompt_len = len(example["prompt"])
+        chosen_len = len(example["chosen"])
+        rejected_len = len(example["rejected"])
+        if (prompt_len + chosen_len) > args.max_seq_length:
+            return False
+        if (rejected_len + chosen_len) > args.max_seq_length:
+            return False
+        return True
+    train_dataset = train_dataset.filter(filter_long_samples)
+    valid_dataset = valid_dataset.filter(filter_long_samples)
+
+    # 配置 DPO 训练参数
+    dpo_config = DPOConfig(
+        output_dir=args.output_model_dir,
+        num_train_epochs=args.num_train_epochs,
+        per_device_train_batch_size=1 if debug_mode else 2,
+        per_device_eval_batch_size=1 if debug_mode else 2,
+        gradient_accumulation_steps=1 if debug_mode else 8,
+        gradient_checkpointing=True,
+        gradient_checkpointing_kwargs={"use_reentrant": False},
+        save_strategy="steps",
+        save_steps=100,
+        save_total_limit=2,
+        logging_steps=10,
+        eval_strategy="steps",  # 添加评估策略
+        eval_steps=100,  # 每100步评估一次
+        learning_rate=args.learning_rate,
+        warmup_steps=100,
+        lr_scheduler_type="cosine",
+        fp16=not is_bfloat16_supported(),  # 如果不支持bfloat16则使用fp16
+        bf16=is_bfloat16_supported(),  # 如果支持bfloat16则使用bf16
+        optim="adamw_8bit",  # 使用8bit优化器节省显存
+        report_to="none",
+        max_length=args.max_seq_length,  # prompt + chosen 的最大长度
+        max_prompt_length=args.max_seq_length // 2,  # prompt 的最大长度
+
+        # DPO 特定参数
+        beta=args.dpo_beta,  # DPO 的温度参数
+        remove_unused_columns=False,
+        dataloader_pin_memory=False,
+
+        # DDP 相关参数
+        ddp_find_unused_parameters=False,  # 重要：告诉DDP忽略未使用的参数
+        local_rank=local_rank,  # 传递当前进程的local_rank
+
+        # 其他优化
+        dataloader_num_workers=args.num_workers or 0,
+    )
+
+    trainer = DPOTrainer(
+        model=model,
+        ref_model=ref_model,
+        args=dpo_config,
+        train_dataset=train_dataset,
+        eval_dataset=valid_dataset,
+        processing_class=tokenizer,
+        # DPOTrainer 会自动处理数据，不需要 data_collator
+    )
+
+    # 开始训练
+    print("开始 DPO 训练...")
+    trainer.train()
+
+    # 保存模型
+    if is_main_process:  # 只在主进程保存
+        print(f"保存模型到: {args.output_model_dir}")
+        # 保存 LoRA 权重
+        model.save_pretrained(args.output_model_dir)
+        tokenizer.save_pretrained(args.output_model_dir)
+        # 如果需要合并并保存完整模型（会占用更多空间）
+        merged_model = model.merge_and_unload()
+        merged_model.save_pretrained(args.output_model_dir + "_merged")
+        print(f"模型保存完成!")
+
+    print("DPO 训练完成!")
+    return
+
+
+if __name__ == "__main__":
+    main()

examples/tutorials/grpo/step_1_download_model_ms.py

@@ -3,9 +3,9 @@
 """
 或使用命令行
 
-python3 step_1_prepare_data.py \
---repo_id qgyd2021/gpt2-for-sequence-classification-sst2-reward \
---local_dir /root/autodl-tmp/OpenMiniMind/trained_models/gpt2-for-sequence-classification-sst2-reward
+python3 step_1_download_model_ms.py \
+--repo_id Qwen/Qwen2.5-3B-Instruct \
+--local_dir /root/autodl-tmp/OpenMiniMind/pretrained_models/Qwen/Qwen2.5-3B-Instruct
 
 """
 import argparse

examples/tutorials/grpo/step_2_train_grpo_model.py

@@ -23,11 +23,8 @@ else:
     temp_directory = Path("/root/autodl-tmp/OpenMiniMind/temp")
 
 from datasets import load_dataset
-import torch
 from transformers import (
-    AutoTokenizer, AutoModelForCausalLM, AutoModelForSequenceClassification,
-    GPT2LMHeadModel, GPT2ForSequenceClassification,
-    DataCollatorWithPadding
+    AutoTokenizer, AutoModelForCausalLM,
 )
 from trl import GRPOConfig, GRPOTrainer
 
@@ -37,8 +34,8 @@ def get_args():
     parser.add_argument(
         "--model_name",
         # default="Qwen/Qwen2.5-3B-Instruct",
-        # default=(project_path / "pretrained_models/Qwen/Qwen2.5-3B-Instruct").as_posix(),
-        default=(project_path / "pretrained_models/jingyaogong/MiniMind2").as_posix(),
+        default=(project_path / "pretrained_models/Qwen/Qwen2.5-3B-Instruct").as_posix(),
+        # default=(project_path / "pretrained_models/jingyaogong/MiniMind2").as_posix(),
         type=str
     )
     parser.add_argument("--dataset_path", default="Jiayi-Pan/Countdown-Tasks-3to4", type=str)
@@ -50,8 +47,6 @@ def get_args():
     # 训练参数
     parser.add_argument("--valid_dataset_size", default=2000, type=int)
 
-    # 生成参数
-
     parser.add_argument(
         "--output_model_dir",
         default=(project_path / "trained_models/qwen2_5-3B-Instruct-Countdown-GRPO").as_posix(),
@@ -277,11 +272,19 @@ def main():
         bf16=False,
         max_grad_norm=1.0,
         report_to="none",  # 可根据需要改为"wandb"等
+
         # GRPO特有参数
         num_generations=4,  # 每个提示生成的响应数量
-        temperature=0.7,
         max_completion_length=512,  # 生成的最大长度
+        steps_per_generation=12,
+        temperature=0.7,
+
+        beta=0.001,
+        num_iterations=1,
+        epsilon=0.2,
         reward_weights=[0.1, 1.0],
+        loss_type="dapo",
+
     )
 
     grpo_trainer = GRPOTrainer(