examples/tutorials/dpo/ultrafeedback-dpo/step_1_prepare_data.py

#!/usr/bin/python3
# -*- coding: utf-8 -*-
"""
或使用命令行
pip install modelscope
modelscope download \
--model 'qgyd2021/Qwen3-8B-sft-deepspeed' \
--local_dir '/root/autodl-tmp/trained_models/Qwen3-8B-sft-deepspeed'

"""
import argparse
import os
from pathlib import Path
import platform

os.environ["HF_ENDPOINT"] = "https://hf-mirror.com"

if platform.system() in ("Windows", "Darwin"):
    from project_settings import project_path, temp_directory
else:
    project_path = os.path.abspath("../../../")
    project_path = Path(project_path)
    temp_directory = Path("/root/autodl-tmp/OpenMiniMind/temp")

from modelscope import snapshot_download
# from huggingface_hub import snapshot_download


def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("--repo_id", default="openai-community/gpt2", type=str)
    parser.add_argument(
        "--local_dir",
        default=(temp_directory / "../trained_models/openai-community/gpt2").as_posix(),
        type=str
    )
    args = parser.parse_args()
    return args


def main():
    args = get_args()

    #modelscope
    snapshot_download(
        model_id=args.repo_id,
        local_dir=args.local_dir,
    )
    #huggingface_hub
    # snapshot_download(
    #     repo_type="model",
    #     repo_id=args.repo_id,
    #     local_dir=args.local_dir,
    # )
    return


if __name__ == "__main__":
    main()