Add files using upload-large-folder tool

ms-swift/examples/train/multi-gpu/device_map/train.sh

@@ -0,0 +1,25 @@
+# 2 * 76GiB
+CUDA_VISIBLE_DEVICES=0,1 \
+MAX_PIXELS=1003520 \
+swift sft \
+    --model Qwen/Qwen2.5-VL-72B-Instruct \
+    --dataset 'modelscope/coco_2014_caption:validation#20000' \
+    --train_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4

ms-swift/examples/train/multimodal/grounding.sh

@@ -0,0 +1,27 @@
+# 20GiB
+# You can refer to `https://github.com/QwenLM/Qwen2.5-VL` for the meaning of the `MAX_PIXELS` parameter.
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+swift sft \
+    --model Qwen/Qwen2-VL-7B-Instruct \
+    --dataset 'AI-ModelScope/coco#20000' \
+    --train_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4

ms-swift/examples/train/multimodal/lora_llm_full_vit/sft.sh

@@ -0,0 +1,30 @@
+# 4 * 22GiB
+# vit/merger lr 1e-5; llm lora lr 1e-4
+NPROC_PER_NODE=4 \
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+MAX_PIXELS=1003520 \
+swift sft \
+    --model Qwen/Qwen2.5-VL-7B-Instruct \
+    --dataset 'AI-ModelScope/coco#20000' \
+    --train_type custom \
+    --optimizer custom \
+    --external_plugins 'examples/train/multimodal/lora_llm_full_vit/custom_plugin.py' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 16 \
+    --lora_alpha 32 \
+    --gradient_accumulation_steps 4 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 8192 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4 \
+    --deepspeed zero2 \
+    --save_only_model true

ms-swift/examples/train/multimodal/rlhf/dpo.sh

@@ -0,0 +1,33 @@
+# 4*50GiB
+# You can refer to `https://github.com/QwenLM/Qwen2.5-VL` for the meaning of the `MAX_PIXELS` parameter.
+# --rlhf_type cpo/orpo/simpo/rm are also supported
+nproc_per_node=2
+
+CUDA_VISIBLE_DEVICES=0,1 \
+NPROC_PER_NODE=$nproc_per_node \
+MAX_PIXELS=1003520 \
+swift rlhf \
+    --rlhf_type dpo \
+    --model Qwen/Qwen2.5-VL-7B-Instruct \
+    --dataset 'swift/RLAIF-V-Dataset#20000' \
+    --train_type lora \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --freeze_vit true \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --deepspeed zero2 \
+    --logging_steps 5 \
+    --max_length 4096 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --dataset_num_proc 4

ms-swift/examples/train/rlhf/ppo.sh

@@ -0,0 +1,33 @@
+# Currently, it only supports the case where the model and reward_model use the same template/tokenizer.
+# Currently, multimodal model PPO is not supported.
+nproc_per_node=4
+
+CUDA_VISIBLE_DEVICES=0,1,2,3 \
+NPROC_PER_NODE=$nproc_per_node \
+swift rlhf \
+    --rlhf_type ppo \
+    --model LLM-Research/Meta-Llama-3.1-8B-Instruct \
+    --reward_model 'AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2' \
+    --train_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#20000' 'AI-ModelScope/alpaca-gpt4-data-en#20000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-5 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps $(expr 16 / $nproc_per_node) \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --deepspeed zero2 \
+    --response_length 512 \
+    --temperature 0.7 \
+    --dataset_num_proc 4

ms-swift/examples/train/seq_cls/qwen2_5/sft.sh

@@ -0,0 +1,28 @@
+# If `num_labels` is provided, it will be considered a classification task,
+# and AutoModelForSequenceClassification will be used to load the model.
+# You can also specify `--model Qwen/Qwen2.5-0.5B-Instruct --use_chat_template true`.
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-0.5B \
+    --train_type lora \
+    --dataset 'DAMO_NLP/jd:cls#2000' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --num_labels 2 \
+    --task_type seq_cls \
+    --use_chat_template false

ms-swift/examples/train/seq_cls/qwen2_vl/infer.sh

@@ -0,0 +1,5 @@
+CUDA_VISIBLE_DEVICES=0 \
+MAX_PIXELS=1003520 \
+swift infer \
+    --adapters output/vx-xxx/checkpoint-xxx \
+    --load_data_args true

ms-swift/examples/train/tuners/adapter/train.sh

@@ -0,0 +1,16 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type adapter \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

ms-swift/examples/train/tuners/boft/train.sh

@@ -0,0 +1,16 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type boft \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

ms-swift/examples/train/tuners/dora/train.sh

@@ -0,0 +1,19 @@
+# 17.2GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type lora \
+    --use_dora true \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

ms-swift/examples/train/tuners/galore/train_galore.sh

@@ -0,0 +1,18 @@
+# 38GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type full \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-5 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot \
+    --use_galore true \
+    --galore_optim_per_parameter true

ms-swift/examples/train/tuners/llamapro/train.sh

@@ -0,0 +1,17 @@
+# 25.4GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type llamapro \
+    --dataset 'swift/self-cognition#1000' \
+    --llamapro_num_new_blocks 4 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

ms-swift/examples/train/tuners/olora/train.sh

@@ -0,0 +1,19 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --init_lora_weights olora \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

ms-swift/examples/train/tuners/pissa/train.sh

@@ -0,0 +1,19 @@
+# 17GiB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --init_lora_weights pissa \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

ms-swift/examples/train/tuners/qlora/train.sh

@@ -0,0 +1,19 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type lora \
+    --dataset 'swift/self-cognition#1000' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot \
+    --quant_bits 4 \
+    --quant_method bnb

ms-swift/examples/train/tuners/reft/train.sh

@@ -0,0 +1,17 @@
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type reft \
+    --dataset 'swift/self-cognition#1000' \
+    --reft_intervention_type 'LoreftIntervention' \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --learning_rate 1e-4 \
+    --gradient_checkpointing false \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 100 \
+    --save_steps 100 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --model_author swift \
+    --model_name swift-robot

ms-swift/ms_swift.egg-info/PKG-INFO

@@ -0,0 +1,545 @@
+Metadata-Version: 2.4
+Name: ms_swift
+Version: 3.5.0.dev0
+Summary: Swift: Scalable lightWeight Infrastructure for Fine-Tuning
+Home-page: https://github.com/modelscope/swift
+Author: DAMO ModelScope teams
+Author-email: contact@modelscope.cn
+License: Apache License 2.0
+Keywords: python,petl,efficient tuners
+Platform: UNKNOWN
+Classifier: Development Status :: 4 - Beta
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: accelerate
+Requires-Dist: addict
+Requires-Dist: aiohttp
+Requires-Dist: attrdict
+Requires-Dist: binpacking
+Requires-Dist: charset_normalizer
+Requires-Dist: cpm_kernels
+Requires-Dist: dacite
+Requires-Dist: datasets<3.4,>=3.0
+Requires-Dist: einops
+Requires-Dist: fastapi
+Requires-Dist: gradio>=3.40.0
+Requires-Dist: importlib_metadata
+Requires-Dist: jieba
+Requires-Dist: matplotlib
+Requires-Dist: modelscope>=1.23
+Requires-Dist: nltk
+Requires-Dist: numpy<2.0
+Requires-Dist: openai
+Requires-Dist: oss2
+Requires-Dist: pandas
+Requires-Dist: peft<0.16,>=0.11
+Requires-Dist: pillow
+Requires-Dist: requests
+Requires-Dist: rouge
+Requires-Dist: safetensors
+Requires-Dist: scipy
+Requires-Dist: sentencepiece
+Requires-Dist: simplejson>=3.3.0
+Requires-Dist: sortedcontainers>=1.5.9
+Requires-Dist: tensorboard
+Requires-Dist: tiktoken
+Requires-Dist: tqdm
+Requires-Dist: transformers<4.53,>=4.33
+Requires-Dist: transformers_stream_generator
+Requires-Dist: trl<0.18,>=0.13
+Requires-Dist: uvicorn
+Requires-Dist: zstandard
+Provides-Extra: eval
+Requires-Dist: evalscope[opencompass]; extra == "eval"
+Requires-Dist: evalscope[vlmeval]; extra == "eval"
+Provides-Extra: swanlab
+Requires-Dist: swanlab; extra == "swanlab"
+Provides-Extra: seq-parallel
+Requires-Dist: xtuner; extra == "seq-parallel"
+Provides-Extra: all
+Requires-Dist: accelerate; extra == "all"
+Requires-Dist: addict; extra == "all"
+Requires-Dist: aiohttp; extra == "all"
+Requires-Dist: attrdict; extra == "all"
+Requires-Dist: binpacking; extra == "all"
+Requires-Dist: charset_normalizer; extra == "all"
+Requires-Dist: cpm_kernels; extra == "all"
+Requires-Dist: dacite; extra == "all"
+Requires-Dist: datasets<3.4,>=3.0; extra == "all"
+Requires-Dist: einops; extra == "all"
+Requires-Dist: fastapi; extra == "all"
+Requires-Dist: gradio>=3.40.0; extra == "all"
+Requires-Dist: importlib_metadata; extra == "all"
+Requires-Dist: jieba; extra == "all"
+Requires-Dist: matplotlib; extra == "all"
+Requires-Dist: modelscope>=1.23; extra == "all"
+Requires-Dist: nltk; extra == "all"
+Requires-Dist: numpy<2.0; extra == "all"
+Requires-Dist: openai; extra == "all"
+Requires-Dist: oss2; extra == "all"
+Requires-Dist: pandas; extra == "all"
+Requires-Dist: peft<0.16,>=0.11; extra == "all"
+Requires-Dist: pillow; extra == "all"
+Requires-Dist: requests; extra == "all"
+Requires-Dist: rouge; extra == "all"
+Requires-Dist: safetensors; extra == "all"
+Requires-Dist: scipy; extra == "all"
+Requires-Dist: sentencepiece; extra == "all"
+Requires-Dist: simplejson>=3.3.0; extra == "all"
+Requires-Dist: sortedcontainers>=1.5.9; extra == "all"
+Requires-Dist: tensorboard; extra == "all"
+Requires-Dist: tiktoken; extra == "all"
+Requires-Dist: tqdm; extra == "all"
+Requires-Dist: transformers<4.53,>=4.33; extra == "all"
+Requires-Dist: transformers_stream_generator; extra == "all"
+Requires-Dist: trl<0.18,>=0.13; extra == "all"
+Requires-Dist: uvicorn; extra == "all"
+Requires-Dist: zstandard; extra == "all"
+Requires-Dist: evalscope[opencompass]; extra == "all"
+Requires-Dist: evalscope[vlmeval]; extra == "all"
+Requires-Dist: xtuner; extra == "all"
+Requires-Dist: swanlab; extra == "all"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: summary
+
+# SWIFT (Scalable lightWeight Infrastructure for Fine-Tuning)
+
+<p align="center">
+    <br>
+    <img src="asset/banner.png"/>
+    <br>
+<p>
+<p align="center">
+<a href="https://modelscope.cn/home">ModelScope Community Website</a>
+<br>
+        <a href="README_CN.md">中文</a> &nbsp ｜ &nbsp English &nbsp
+</p>
+
+<p align="center">
+<img src="https://img.shields.io/badge/python-3.10-5be.svg">
+<img src="https://img.shields.io/badge/pytorch-%E2%89%A52.0-orange.svg">
+<a href="https://github.com/modelscope/modelscope/"><img src="https://img.shields.io/badge/modelscope-%E2%89%A51.19-5D91D4.svg"></a>
+<a href="https://pypi.org/project/ms-swift/"><img src="https://badge.fury.io/py/ms-swift.svg"></a>
+<a href="https://github.com/modelscope/swift/blob/main/LICENSE"><img src="https://img.shields.io/github/license/modelscope/swift"></a>
+<a href="https://pepy.tech/project/ms-swift"><img src="https://pepy.tech/badge/ms-swift"></a>
+<a href="https://github.com/modelscope/swift/pulls"><img src="https://img.shields.io/badge/PR-welcome-55EB99.svg"></a>
+</p>
+
+<p align="center">
+<a href="https://trendshift.io/repositories/6427" target="_blank"><img src="https://trendshift.io/api/badge/repositories/6427" alt="modelscope%2Fswift | Trendshift" style="width: 250px; height: 55px;" width="250" height="55"/></a>
+</p>
+
+<p align="center">
+        <a href="https://arxiv.org/abs/2408.05517">Paper</a> &nbsp ｜ <a href="https://swift.readthedocs.io/en/latest/">English Documentation</a> &nbsp ｜ &nbsp <a href="https://swift.readthedocs.io/zh-cn/latest/">中文文档</a> &nbsp
+</p>
+
+## 📖 Table of Contents
+- [Groups](#-Groups)
+- [Introduction](#-introduction)
+- [News](#-news)
+- [Installation](#%EF%B8%8F-installation)
+- [Quick Start](#-quick-Start)
+- [Usage](#-Usage)
+- [License](#-License)
+- [Citation](#-citation)
+
+
+## ☎ Groups
+
+You can contact us and communicate with us by adding our group:
+
+
+[Discord Group](https://discord.com/invite/D27yfEFVz5)              |  WeChat Group
+:-------------------------:|:-------------------------:
+<img src="asset/discord_qr.jpg" width="200" height="200">  |  <img src="asset/wechat.png" width="200" height="200">
+
+
+## 📝 Introduction
+🍲 ms-swift is an official framework provided by the ModelScope community for fine-tuning and deploying large language models and multi-modal large models. It currently supports the training (pre-training, fine-tuning, human alignment), inference, evaluation, quantization, and deployment of 500+ large models and 200+ multi-modal large models. These large language models (LLMs) include models such as Qwen3, Qwen3-MoE, Qwen2.5, InternLM3, GLM4, Mistral, DeepSeek-R1, Yi1.5, TeleChat2, Baichuan2, and Gemma2. The multi-modal LLMs include models such as Qwen2.5-VL, Qwen2-Audio, Llama3.4, Llava, InternVL2.5, MiniCPM-V-2.6, GLM4v, Xcomposer2.5, Yi-VL, DeepSeek-VL2, Phi3.5-Vision, and GOT-OCR2.
+
+🍔 Additionally, ms-swift incorporates the latest training technologies, including lightweight techniques such as LoRA, QLoRA, Llama-Pro, LongLoRA, GaLore, Q-GaLore, LoRA+, LISA, DoRA, FourierFt, ReFT, UnSloth, and Liger, as well as human alignment training methods like DPO, GRPO, RM, PPO, KTO, CPO, SimPO, and ORPO. ms-swift supports acceleration of inference, evaluation, and deployment modules using vLLM and LMDeploy, and it supports model quantization with technologies like GPTQ, AWQ, and BNB. Furthermore, ms-swift offers a Gradio-based Web UI and a wealth of best practices.
+
+**Why choose ms-swift?**
+
+- 🍎 **Model Types**: Supports 500+ pure text large models, **200+ multi-modal large models**, as well as All-to-All multi-modal models, sequence classification models, and embedding models, **covering the entire process from training to deployment**.
+- **Dataset Types**: Comes with 150+ pre-training, fine-tuning, human alignment, multi-modal datasets, and supports custom datasets.
+- **Hardware Support**: Compatible with CPU, RTX series, T4/V100, A10/A100/H100, Ascend NPU, MPS, etc.
+- 🍊 **Lightweight Training**: Supports lightweight fine-tuning methods like LoRA, QLoRA, DoRA, LoRA+, ReFT, RS-LoRA, LLaMAPro, Adapter, GaLore, Q-Galore, LISA, UnSloth, Liger-Kernel.
+- **Distributed Training**: Supports distributed data parallel (DDP), device_map simple model parallelism, DeepSpeed ZeRO2/ZeRO3, FSDP, and other distributed training techniques.
+- **Quantization Training**: Supports training quantized models like BNB, AWQ, GPTQ, AQLM, HQQ, EETQ.
+- **RLHF Training**: Supports human alignment training methods such as DPO, GRPO, RM, PPO, KTO, CPO, SimPO, ORPO for both pure text and multi-modal large models.
+- 🍓 **Multi-Modal Training**: Supports training on different modalities like images, videos, and audio, for tasks like VQA, captioning, OCR, and grounding.
+- **Interface Training**: Provides capabilities for training, inference, evaluation, quantization through an interface, completing the whole large model pipeline.
+- **Plugin and Extension**: Supports custom model and dataset extensions, as well as customization of components like loss, metric, trainer, loss-scale, callback, optimizer.
+- 🍉 **Toolbox Capabilities**: Offers not only training support for large models and multi-modal large models but also covers the entire process of inference, evaluation, quantization, and deployment.
+- **Inference Acceleration**: Supports inference acceleration engines like PyTorch, vLLM, LmDeploy, and provides OpenAI API for accelerating inference, deployment, and evaluation modules.
+- **Model Evaluation**: Uses EvalScope as the evaluation backend and supports evaluation on 100+ datasets for both pure text and multi-modal models.
+- **Model Quantization**: Supports AWQ, GPTQ, and BNB quantized exports, with models that can use vLLM/LmDeploy for inference acceleration and continue training.
+
+
+## 🎉 News
+- ��� 2025.05.11: GRPO now supports custom processing logic for reward models. See the GenRM example [here](./docs/source_en/Instruction/GRPO.md#customized-reward-models) .
+- 🎁 2025.04.15: The ms-swift paper has been accepted by AAAI 2025. You can find the paper at [this link](https://ojs.aaai.org/index.php/AAAI/article/view/35383).
+- 🎁 2025.03.23: Multi-round GRPO is now supported for training multi-turn dialogue scenarios (e.g., agent tool calling). Please refer to the [training script](https://idealab.alibaba-inc.com/examples/train/grpo/internal/train_multi_round.sh).
+- 🎁 2025.03.16: Support for Megatron's parallel training techniques is now available. Please see the [Megatron-SWIFT training documentation](https://swift.readthedocs.io/zh-cn/latest/Instruction/Megatron-SWIFT训练.html).
+- 🎁 2025.03.15: Fine-tuning of embedding models for both pure text and multimodal models is supported. Please check the [training script](https://idealab.alibaba-inc.com/examples/train/embedding).
+- 🎁 2025.03.05: The hybrid mode for GRPO is supported, with a script for training a 72B model on 4 GPUs (4*80G) available [here](https://idealab.alibaba-inc.com/examples/train/grpo/internal/train_72b_4gpu.sh). Tensor parallelism with vllm is also supported, with the training script available [here](https://idealab.alibaba-inc.com/examples/train/grpo/internal/multi_gpu_mp_colocate.sh).
+- 🎁 2025.02.21: The GRPO algorithm now supports LMDeploy, with the training script available [here](https://idealab.alibaba-inc.com/examples/train/grpo/internal/full_lmdeploy.sh). Additionally, the performance of the GRPO algorithm has been tested, achieving a training speed increase of up to 300% using various tricks. Please check the WanDB table [here](https://wandb.ai/tastelikefeet/grpo_perf_test?nw=nwuseryuzezyz).
+- 🎁 2025.02.21: The `swift sample` command is now supported. The reinforcement fine-tuning script can be found [here](https://idealab.alibaba-inc.com/docs/source/Instruction/强化微调.md), and the large model API distillation sampling script is available [here](https://idealab.alibaba-inc.com/examples/sampler/distill/distill.sh).
+- 🔥 2025.02.12: Support for the GRPO (Group Relative Policy Optimization) training algorithm has been added. Documentation is available [here](https://idealab.alibaba-inc.com/docs/source/Instruction/GRPO.md).
+- 🎁 2024.12.04: Major update to **ms-swift 3.0**. Please refer to the [release notes and changes](https://swift.readthedocs.io/zh-cn/latest/Instruction/ReleaseNote3.0.html).
+<details><summary>More</summary>
+
+- 🎉 2024.08.12: The ms-swift paper has been published on arXiv and can be read [here](https://arxiv.org/abs/2408.05517).
+- 🔥 2024.08.05: Support for using [evalscope](https://github.com/modelscope/evalscope/) as a backend for evaluating large models and multimodal models.
+- 🔥 2024.07.29: Support for using [vllm](https://github.com/vllm-project/vllm) and [lmdeploy](https://github.com/InternLM/lmdeploy) to accelerate inference for large models and multimodal models. When performing infer/deploy/eval, you can specify `--infer_backend vllm/lmdeploy`.
+- 🔥 2024.07.24: Support for human preference alignment training for multimodal large models, including DPO/ORPO/SimPO/CPO/KTO/RM/PPO.
+- 🔥 2024.02.01: Support for Agent training! The training algorithm is derived from [this paper](https://arxiv.org/pdf/2309.00986.pdf).
+</details>
+
+## 🛠️ Installation
+To install using pip:
+```shell
+pip install ms-swift -U
+```
+
+To install from source:
+```shell
+# pip install git+https://github.com/modelscope/ms-swift.git
+
+git clone https://github.com/modelscope/ms-swift.git
+cd ms-swift
+pip install -e .
+```
+
+Running Environment:
+
+|              | Range        | Recommended | Notes                                     |
+| ------------ |--------------| ----------- | ----------------------------------------- |
+| python       | >=3.9        | 3.10        |                                           |
+| cuda         |              | cuda12      | No need to install if using CPU, NPU, MPS |
+| torch        | >=2.0        |             |                                           |
+| transformers | >=4.33       | 4.51      |                                           |
+| modelscope   | >=1.23       |             |                                           |
+| peft | >=0.11,<0.16 | ||
+| trl | >=0.13,<0.18 | 0.17 |RLHF|
+| deepspeed    | >=0.14       | 0.14.5 | Training                                  |
+| vllm         | >=0.5.1      | 0.7.3/0.8       | Inference/Deployment/Evaluation           |
+| lmdeploy     | >=0.5        | 0.8       | Inference/Deployment/Evaluation           |
+| evalscope | >=0.11       |  | Evaluation |
+
+For more optional dependencies, you can refer to [here](https://github.com/modelscope/ms-swift/blob/main/requirements/install_all.sh).
+
+
+## 🚀 Quick Start
+
+10 minutes of self-cognition fine-tuning of Qwen2.5-7B-Instruct on a single 3090 GPU:
+
+### Command Line Interface
+
+```shell
+# 22GB
+CUDA_VISIBLE_DEVICES=0 \
+swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --train_type lora \
+    --dataset 'AI-ModelScope/alpaca-gpt4-data-zh#500' \
+              'AI-ModelScope/alpaca-gpt4-data-en#500' \
+              'swift/self-cognition#500' \
+    --torch_dtype bfloat16 \
+    --num_train_epochs 1 \
+    --per_device_train_batch_size 1 \
+    --per_device_eval_batch_size 1 \
+    --learning_rate 1e-4 \
+    --lora_rank 8 \
+    --lora_alpha 32 \
+    --target_modules all-linear \
+    --gradient_accumulation_steps 16 \
+    --eval_steps 50 \
+    --save_steps 50 \
+    --save_total_limit 2 \
+    --logging_steps 5 \
+    --max_length 2048 \
+    --output_dir output \
+    --system 'You are a helpful assistant.' \
+    --warmup_ratio 0.05 \
+    --dataloader_num_workers 4 \
+    --model_author swift \
+    --model_name swift-robot
+```
+
+Tips:
+
+- If you want to train with a custom dataset, you can refer to [this guide](https://swift.readthedocs.io/en/latest/Customization/Custom-dataset.html) to organize your dataset format and specify `--dataset <dataset_path>`.
+- The `--model_author` and `--model_name` parameters are only effective when the dataset includes `swift/self-cognition`.
+- To train with a different model, simply modify `--model <model_id/model_path>`.
+- By default, ModelScope is used for downloading models and datasets. If you want to use HuggingFace, simply specify `--use_hf true`.
+
+After training is complete, use the following command to infer with the trained weights:
+
+- Here, `--adapters` should be replaced with the last checkpoint folder generated during training. Since the adapters folder contains the training parameter file `args.json`, there is no need to specify `--model`, `--system` separately; Swift will automatically read these parameters. To disable this behavior, you can set `--load_args false`.
+
+```shell
+# Using an interactive command line for inference.
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --adapters output/vx-xxx/checkpoint-xxx \
+    --stream true \
+    --temperature 0 \
+    --max_new_tokens 2048
+
+# merge-lora and use vLLM for inference acceleration
+CUDA_VISIBLE_DEVICES=0 \
+swift infer \
+    --adapters output/vx-xxx/checkpoint-xxx \
+    --stream true \
+    --merge_lora true \
+    --infer_backend vllm \
+    --max_model_len 8192 \
+    --temperature 0 \
+    --max_new_tokens 2048
+```
+
+Finally, use the following command to push the model to ModelScope:
+
+```shell
+CUDA_VISIBLE_DEVICES=0 \
+swift export \
+    --adapters output/vx-xxx/checkpoint-xxx \
+    --push_to_hub true \
+    --hub_model_id '<your-model-id>' \
+    --hub_token '<your-sdk-token>' \
+    --use_hf false
+```
+
+
+### Web-UI
+The Web-UI is a **zero-threshold** training and deployment interface solution based on Gradio interface technology. For more details, you can check [here](https://swift.readthedocs.io/en/latest/GetStarted/Web-UI.html).
+
+```shell
+SWIFT_UI_LANG=en swift web-ui
+```
+
+![image.png](./docs/resources/web-ui-en.jpg)
+
+### Using Python
+
+ms-swift also supports training and inference using Python. Below is pseudocode for training and inference. For more details, you can refer to [here](https://github.com/modelscope/ms-swift/blob/main/examples/notebook/qwen2_5-self-cognition/self-cognition-sft.ipynb).
+
+Training:
+
+```python
+# Retrieve the model and template, and add a trainable LoRA module
+model, tokenizer = get_model_tokenizer(model_id_or_path, ...)
+template = get_template(model.model_meta.template, tokenizer, ...)
+model = Swift.prepare_model(model, lora_config)
+
+# Download and load the dataset, and encode the text into tokens
+train_dataset, val_dataset = load_dataset(dataset_id_or_path, ...)
+train_dataset = EncodePreprocessor(template=template)(train_dataset, num_proc=num_proc)
+val_dataset = EncodePreprocessor(template=template)(val_dataset, num_proc=num_proc)
+
+# Train the model
+trainer = Seq2SeqTrainer(
+    model=model,
+    args=training_args,
+    data_collator=template.data_collator,
+    train_dataset=train_dataset,
+    eval_dataset=val_dataset,
+    template=template,
+)
+trainer.train()
+```
+Inference:
+
+```python
+# Perform inference using the native PyTorch engine
+engine = PtEngine(model_id_or_path, adapters=[lora_checkpoint])
+infer_request = InferRequest(messages=[{'role': 'user', 'content': 'who are you?'}])
+request_config = RequestConfig(max_tokens=max_new_tokens, temperature=temperature)
+
+resp_list = engine.infer([infer_request], request_config)
+print(f'response: {resp_list[0].choices[0].message.content}')
+```
+
+## ✨ Usage
+Here is a minimal example of training to deployment using ms-swift. For more details, you can check the [examples](https://github.com/modelscope/ms-swift/tree/main/examples).
+
+- If you want to use other models or datasets (including multimodal models and datasets), you only need to modify `--model` to specify the corresponding model's ID or path, and modify `--dataset` to specify the corresponding dataset's ID or path.
+- By default, ModelScope is used for downloading models and datasets. If you want to use HuggingFace, simply specify `--use_hf true`.
+
+|   Useful Links |
+| ------ |
+|   [🔥Command Line Parameters](https://swift.readthedocs.io/en/latest/Instruction/Command-line-parameters.html)   |
+|   [Supported Models and Datasets](https://swift.readthedocs.io/en/latest/Instruction/Supported-models-and-datasets.html)   |
+|   [Custom Models](https://swift.readthedocs.io/en/latest/Customization/Custom-model.html), [🔥Custom Datasets](https://swift.readthedocs.io/en/latest/Customization/Custom-dataset.html)   |
+|   [LLM Tutorial](https://github.com/modelscope/modelscope-classroom/tree/main/LLM-tutorial)   |
+
+### Training
+
+Supported Training Methods:
+
+| Method                             | Full-Parameter                                               | LoRA                                                                                        | QLoRA                                                        | Deepspeed                                                    | Multi-Node                                                   | Multi-Modal                                                                                  |
+|------------------------------------|--------------------------------------------------------------|---------------------------------------------------------------------------------------------|--------------------------------------------------------------|--------------------------------------------------------------|--------------------------------------------------------------|----------------------------------------------------------------------------------------------|
+| Pre-training                       | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/pretrain/train.sh) | ✅                                                                                           | ✅                                                            | ✅                                                            | ✅                                                            | ✅                                                                                            |
+| Instruction Supervised Fine-tuning | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/full/train.sh) | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/lora_sft.sh)            | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/qlora) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-gpu/deepspeed) | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multi-node)                                                            | [✅](https://github.com/modelscope/ms-swift/tree/main/examples/train/multimodal)              |
+| DPO Training                       | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/dpo.sh)            | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/dpo.sh) | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/dpo.sh)  |
+| GRPO Training                      | [✅]((https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo/internal/grpo_zero2.sh)) | ✅                                                                                           | ✅                                                            | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/grpo/internal/multi_node)                                    | ✅                                                                                            |
+| Reward Model Training              | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/rm.sh)             | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/rm.sh) | ✅                                                            | ✅                                                                                            |
+| PPO Training                       | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/ppo.sh)            | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/ppo.sh) | ✅                                                            | ❌                                                                                            |
+| KTO Training                       | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/kto.sh)            | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/kto.sh) | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/multimodal/rlhf/kto.sh)  |
+| CPO Training                       | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/cpo.sh)            | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/cpo.sh) | ✅                                                            | ✅                                                                                            |
+| SimPO Training                     | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/simpo.sh)          | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/simpo.sh) | ✅                                                            | ✅                                                                                            |
+| ORPO Training                      | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/orpo.sh)           | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/rlhf/orpo.sh) | ✅                                                            | ✅                                                                                            |
+| Classification Model Training      | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/seq_cls/qwen2_5/sft.sh) | ✅                                                            | ✅                                                            | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/seq_cls/qwen2_vl/sft.sh) |
+| Embedding Model Training           | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/embedding/train_gte.sh) | ✅                                                            | ✅                                                            | ✅                                                            | [✅](https://github.com/modelscope/ms-swift/blob/main/examples/train/embedding/train_gme.sh)  |
+
+
+
+Pre-training:
+```shell
+# 8*A100
+NPROC_PER_NODE=8 \
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 \
+swift pt \
+    --model Qwen/Qwen2.5-7B \
+    --dataset swift/chinese-c4 \
+    --streaming true \
+    --train_type full \
+    --deepspeed zero2 \
+    --output_dir output \
+    --max_steps 10000 \
+    ...
+```
+
+Fine-tuning:
+```shell
+CUDA_VISIBLE_DEVICES=0 swift sft \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --dataset AI-ModelScope/alpaca-gpt4-data-en \
+    --train_type lora \
+    --output_dir output \
+    ...
+```
+
+RLHF:
+```shell
+CUDA_VISIBLE_DEVICES=0 swift rlhf \
+    --rlhf_type dpo \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --dataset hjh0119/shareAI-Llama3-DPO-zh-en-emoji \
+    --train_type lora \
+    --output_dir output \
+    ...
+```
+
+
+### Inference
+```shell
+CUDA_VISIBLE_DEVICES=0 swift infer \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --stream true \
+    --infer_backend pt \
+    --max_new_tokens 2048
+
+# LoRA
+CUDA_VISIBLE_DEVICES=0 swift infer \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --adapters swift/test_lora \
+    --stream true \
+    --infer_backend pt \
+    --temperature 0 \
+    --max_new_tokens 2048
+```
+
+### Interface Inference
+```shell
+CUDA_VISIBLE_DEVICES=0 swift app \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --stream true \
+    --infer_backend pt \
+    --max_new_tokens 2048
+```
+
+### Deployment
+```shell
+CUDA_VISIBLE_DEVICES=0 swift deploy \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend vllm
+```
+
+### Sampling
+```shell
+CUDA_VISIBLE_DEVICES=0 swift sample \
+    --model LLM-Research/Meta-Llama-3.1-8B-Instruct \
+    --sampler_engine pt \
+    --num_return_sequences 5 \
+    --dataset AI-ModelScope/alpaca-gpt4-data-zh#5
+```
+
+### Evaluation
+```shell
+CUDA_VISIBLE_DEVICES=0 swift eval \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --infer_backend lmdeploy \
+    --eval_backend OpenCompass \
+    --eval_dataset ARC_c
+```
+
+### Quantization
+```shell
+CUDA_VISIBLE_DEVICES=0 swift export \
+    --model Qwen/Qwen2.5-7B-Instruct \
+    --quant_bits 4 --quant_method awq \
+    --dataset AI-ModelScope/alpaca-gpt4-data-zh \
+    --output_dir Qwen2.5-7B-Instruct-AWQ
+```
+
+### Push Model
+```shell
+swift export \
+    --model <model-path> \
+    --push_to_hub true \
+    --hub_model_id '<model-id>' \
+    --hub_token '<sdk-token>'
+```
+
+## 🏛 License
+
+This framework is licensed under the [Apache License (Version 2.0)](https://github.com/modelscope/modelscope/blob/master/LICENSE). For models and datasets, please refer to the original resource page and follow the corresponding License.
+
+## 📎 Citation
+
+```bibtex
+@misc{zhao2024swiftascalablelightweightinfrastructure,
+      title={SWIFT:A Scalable lightWeight Infrastructure for Fine-Tuning},
+      author={Yuze Zhao and Jintao Huang and Jinghan Hu and Xingjun Wang and Yunlin Mao and Daoze Zhang and Zeyinzi Jiang and Zhikai Wu and Baole Ai and Ang Wang and Wenmeng Zhou and Yingda Chen},
+      year={2024},
+      eprint={2408.05517},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2408.05517},
+}
+```
+
+## Star History
+
+[![Star History Chart](https://api.star-history.com/svg?repos=modelscope/swift&type=Date)](https://star-history.com/#modelscope/ms-swift&Date)

ms-swift/ms_swift.egg-info/not-zip-safe

@@ -0,0 +1 @@
+

ms-swift/requirements/install_all.sh

@@ -0,0 +1,12 @@
+# please use python=3.10, cuda12.*
+# sh requirements/install_all.sh
+pip install "vllm>=0.5.1" -U
+pip install "lmdeploy>=0.5" -U --no-deps
+pip install autoawq -U --no-deps
+pip install auto_gptq optimum bitsandbytes -U
+pip install git+https://github.com/modelscope/ms-swift.git
+pip install timm -U
+pip install deepspeed -U
+pip install qwen_vl_utils qwen_omni_utils decord librosa pyav icecream soundfile -U
+pip install liger_kernel nvitop pre-commit -U
+# flash-attn: https://github.com/Dao-AILab/flash-attention/releases

ms-swift/requirements/seq_parallel.txt

@@ -0,0 +1 @@
+xtuner

ms-swift/requirements/swanlab.txt

@@ -0,0 +1 @@
+swanlab

ms-swift/scripts/benchmark/config/tuner.json

@@ -0,0 +1,301 @@
+{
+    "cmd": "sft",
+    "requirements":{
+        "gpu": "1",
+        "ddp": "1"
+    },
+    "eval_requirements": {
+      "gpu": "1"
+    },
+    "eval_dataset": ["ceval", "gsm8k", "arc"],
+    "args": {
+      "model": "Qwen/Qwen-7B-Chat",
+      "dataset": "iic/ms_agent",
+      "per_device_train_batch_size": 1,
+      "max_length": 2048,
+      "loss_scale": "react",
+      "gradient_accumulation_steps": 16,
+      "learning_rate": 5e-5,
+      "attn_impl": "flash_attn",
+      "eval_steps": 2000,
+      "save_steps": 2000,
+      "num_train_epochs": 2,
+      "gradient_checkpointing": true,
+      "weight_decay": 0.01,
+      "warmup_ratio": 0.03,
+      "save_total_limit": 2,
+      "logging_steps": 10
+    },
+    "experiment": [
+      {
+        "name": "lora",
+        "args": {
+          "train_type": "lora",
+          "lora_rank": 8,
+          "lora_alpha": 32
+        }
+      },
+      {
+        "name": "lora+packing",
+        "args": {
+          "train_type": "lora",
+          "lora_rank": 8,
+          "lora_alpha": 32,
+          "packing": true,
+          "eval_steps": 200,
+          "save_steps": 200
+        }
+      },
+      {
+        "name": "lora+packing+ddp",
+        "requirements":{
+          "gpu": "2",
+          "ddp": "2"
+        },
+        "args": {
+          "train_type": "lora",
+          "lora_rank": 8,
+          "lora_alpha": 32,
+          "packing": true,
+          "eval_steps": 100,
+          "save_steps": 100
+        }
+      },
+      {
+        "name": "lora+packing+lazytokenize",
+        "args": {
+          "train_type": "lora",
+          "lora_rank": 8,
+          "lora_alpha": 32,
+          "packing": true,
+          "lazy_tokenize": true,
+          "eval_steps": 200,
+          "save_steps": 200
+        }
+      },
+      {
+        "name": "lora+",
+        "args": {
+          "train_type": "lora",
+          "lora_rank": 8,
+          "lora_alpha": 32,
+          "lorap_lr_ratio": 16.0
+        }
+      },
+      {
+        "name": "rslora",
+        "args": {
+          "train_type": "lora",
+          "lora_rank": 8,
+          "lora_alpha": 32,
+          "use_rslora": true
+        }
+      },
+      {
+        "name": "dora",
+        "args": {
+          "train_type": "lora",
+          "lora_rank": 8,
+          "lora_alpha": 32,
+          "use_dora": true
+        }
+      },
+      {
+        "name": "lora+neftune",
+        "args": {
+          "train_type": "lora",
+          "lora_rank": 8,
+          "lora_alpha": 32,
+          "neftune_noise_alpha": 15.0
+        }
+      },
+      {
+        "name": "llamapro",
+        "args": {
+          "train_type": "llamapro",
+          "llamapro_num_new_blocks": "4"
+        }
+      },
+      {
+        "name": "full",
+        "requirements":{
+          "gpu": "1",
+          "ddp": "1"
+        },
+        "args": {
+          "train_type": "full"
+        }
+      },
+      {
+        "name": "reft",
+        "requirements":{
+          "gpu": "1",
+          "ddp": "1"
+        },
+        "args": {
+          "train_type": "reft",
+          "gradient_checkpointing": "false",
+          "loss_scale": "default"
+        }
+      },
+      {
+        "name": "full+galore128+quantize",
+        "requirements":{
+          "gpu": "1",
+          "ddp": "1"
+        },
+        "args": {
+          "train_type": "full",
+          "use_galore": "true",
+          "galore_rank": "128",
+          "galore_update_proj_gap": "200",
+          "galore_optim_per_parameter": "false",
+          "galore_with_embedding": "false",
+          "galore_quantization": "true"
+        }
+      },
+      {
+        "name": "full+galore128+quantize+proj_quant",
+        "requirements":{
+          "gpu": "1",
+          "ddp": "1"
+        },
+        "args": {
+          "train_type": "full",
+          "use_galore": "true",
+          "galore_rank": "128",
+          "galore_update_proj_gap": "200",
+          "galore_optim_per_parameter": "false",
+          "galore_with_embedding": "false",
+          "galore_quantization": "true",
+          "galore_proj_quant": "true"
+        }
+      },
+      {
+        "name": "full+galore128",
+        "requirements":{
+          "gpu": "1",
+          "ddp": "1"
+        },
+        "args": {
+          "train_type": "full",
+          "use_galore": "true",
+          "galore_rank": "128",
+          "galore_update_proj_gap": "200",
+          "galore_optim_per_parameter": "false",
+          "galore_with_embedding": "false"
+        }
+      },
+      {
+        "name": "full+galore64",
+        "requirements":{
+          "gpu": "1",
+          "ddp": "1"
+        },
+        "args": {
+          "train_type": "full",
+          "use_galore": "true",
+          "galore_rank": "64",
+          "galore_update_proj_gap": "200",
+          "galore_optim_per_parameter": "false",
+          "galore_with_embedding": "false"
+        }
+      },
+      {
+        "name": "full+galore32",
+        "requirements":{
+          "gpu": "1",
+          "ddp": "1"
+        },
+        "args": {
+          "train_type": "full",
+          "use_galore": "true",
+          "galore_rank": "32",
+          "galore_update_proj_gap": "200",
+          "galore_optim_per_parameter": "false",
+          "galore_with_embedding": "false"
+        }
+      },
+      {
+        "name": "full+galore_emb",
+        "requirements":{
+          "gpu": "1",
+          "ddp": "1"
+        },
+        "args": {
+          "train_type": "full",
+          "use_galore": "true",
+          "galore_rank": "128",
+          "galore_update_proj_gap": "200",
+          "galore_optim_per_parameter": "false",
+          "galore_with_embedding": "true"
+        }
+      },
+      {
+        "name": "full+galore_perparam",
+        "requirements":{
+          "gpu": "1",
+          "ddp": "1"
+        },
+        "args": {
+          "train_type": "full",
+          "use_galore": "true",
+          "galore_rank": "128",
+          "galore_update_proj_gap": "200",
+          "galore_optim_per_parameter": "true",
+          "galore_with_embedding": "false"
+        }
+      },
+      {
+        "name": "adalora",
+        "args": {
+          "train_type": "adalora",
+          "lora_rank": 8,
+          "lora_alpha": 32
+        }
+      },
+      {
+        "name": "adapter",
+        "args": {
+          "train_type": "adapter"
+        }
+      },
+      {
+        "name": "full+lisa_2",
+        "info": "lisa 2layers + full",
+        "args": {
+          "train_type": "full",
+          "lisa_activated_layers": 2,
+          "lisa_step_interval": 20
+        }
+      },
+      {
+        "name": "full+lisa_4",
+        "info": "lisa 4layers + full",
+        "args": {
+          "train_type": "full",
+          "lisa_activated_layers": 4,
+          "lisa_step_interval": 20
+        }
+      },
+      {
+        "name": "unsloth+lora+q4",
+        "info": "unsloth lora quantization bit 4",
+        "args": {
+          "train_type": "lora",
+          "tuner_backend": "unsloth",
+          "quantization_bit": 4,
+          "model": "LLM-Research/Meta-Llama-3-8B-Instruct"
+        }
+      },
+      {
+        "name": "unsloth+full",
+        "info": "unsloth full",
+        "args": {
+          "train_type": "full",
+          "tuner_backend": "unsloth",
+          "model_type": "LLM-Research/Meta-Llama-3-8B-Instruct"
+        }
+      }
+    ]
+}

ms-swift/scripts/benchmark/exp.py

@@ -0,0 +1,50 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import argparse
+import os
+import os.path
+
+from exp_utils import ExpManager, find_all_config
+
+from swift.utils import *
+
+logger = get_logger()
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Simple args for swift experiments.')
+    parser.add_argument(
+        '--config',
+        type=str,
+        default=None,
+        required=True,
+        help='The experiment config file',
+    )
+    parser.add_argument(
+        '--save_dir',
+        type=str,
+        default='./experiment',
+        required=False,
+        help='The experiment output folder',
+    )
+
+    args = parser.parse_args()
+    return args
+
+
+def llm_exp():
+    args = parse_args()
+    config: str = args.config
+    config = config.split(',')
+    os.makedirs(args.save_dir, exist_ok=True)
+    all_configs = []
+    if not isinstance(config, list):
+        config = [config]
+    for dir_or_file in config:
+        all_configs.extend(find_all_config(dir_or_file))
+    args.config = all_configs
+    exp_manager = ExpManager()
+    exp_manager.begin(args)
+
+
+if __name__ == '__main__':
+    llm_exp()

ms-swift/scripts/benchmark/generate_report.py

@@ -0,0 +1,433 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import dataclasses
+import os
+from dataclasses import dataclass
+from typing import Any, Dict, List
+
+import json
+import numpy as np
+
+from swift.llm.template import split_str_parts_by
+
+
+@dataclass
+class ModelOutput:
+
+    group: str = None
+
+    name: str = None
+
+    cmd: str = None
+
+    requirements: Dict[str, str] = dataclasses.field(default_factory=dict)
+
+    args: Dict[str, Any] = dataclasses.field(default_factory=dict)
+
+    memory: str = None
+
+    train_time: float = None
+
+    train_samples: int = None
+
+    train_samples_per_second: float = None
+
+    last_model_checkpoint: str = None
+
+    best_model_checkpoint: str = None
+
+    best_metric: Any = None
+
+    global_step: int = None
+
+    num_total_parameters: float = None
+
+    num_trainable_parameters: float = None
+
+    num_buffers: float = None
+
+    trainable_parameters_percentage: float = None
+
+    train_dataset_info: str = None
+
+    val_dataset_info: str = None
+
+    train_create_time: float = None
+
+    eval_tokens: int = None
+
+    eval_time: float = None
+
+    reports: Dict[str, Any] = None
+
+    train_loss: float = None
+
+    @property
+    def tuner_hyper_params(self):
+        hyper_params = ''
+        args = self.args
+        if 'sft_type' not in args:
+            return ''
+        if args['sft_type'] in ('lora', 'adalora', 'longlora'):
+            if 'lora_rank' in args:
+                hyper_params += f'rank={args["lora_rank"]}/' \
+                                f'target={args["lora_target_modules"]}/' \
+                                f'alpha={args["lora_alpha"]}/' \
+                                f'lr_ratio={args.get("lora_lr_ratio", None)}/' \
+                                f'use_rslora={args.get("use_rslora", False)}/' \
+                                f'use_dora={args.get("use_dora", False)}'
+            else:
+                hyper_params = ''
+        if args['sft_type'] == 'full':
+            if 'use_galore' in args and args['use_galore'] == 'true':
+                hyper_params += f'galore_rank={args["galore_rank"]}/' \
+                                f'galore_per_parameter={args["galore_optim_per_parameter"]}/' \
+                                f'galore_with_embedding={args["galore_with_embedding"]}/'
+        if args['sft_type'] == 'llamapro':
+            hyper_params += f'num_blocks={args["llamapro_num_new_blocks"]}/'
+        if 'neftune_noise_alpha' in args and args['neftune_noise_alpha']:
+            hyper_params += f'neftune_noise_alpha={args["neftune_noise_alpha"]}/'
+
+        if hyper_params.endswith('/'):
+            hyper_params = hyper_params[:-1]
+        return hyper_params
+
+    @property
+    def hyper_parameters(self):
+        if 'learning_rate' not in self.args:
+            return ''
+        return f'lr={self.args["learning_rate"]}/' \
+               f'epoch={self.args["num_train_epochs"]}'
+
+    @property
+    def train_speed(self):
+        if self.train_samples_per_second:
+            return f'{self.train_samples_per_second:.2f}({self.train_samples} samples/{self.train_time:.2f} seconds)'
+        else:
+            return ''
+
+    @property
+    def infer_speed(self):
+        if self.eval_tokens:
+            return f'{self.eval_tokens / self.eval_time:.2f}({self.eval_tokens} tokens/{self.eval_time:.2f} seconds)'
+        return ''
+
+
+def generate_sft_report(outputs: List[ModelOutput]):
+    gsm8k_accs = []
+    arc_accs = []
+    ceval_accs = []
+    for output in outputs:
+        gsm8k_acc = None
+        arc_acc = None
+        ceval_acc = None
+        for report in (output.reports or []):
+            if report['name'] == 'gsm8k':
+                gsm8k_acc = report['score']
+            if report['name'] == 'arc':
+                arc_acc = report['score']
+            if report['name'] == 'ceval':
+                ceval_acc = report['score']
+        gsm8k_accs.append(gsm8k_acc)
+        arc_accs.append(arc_acc)
+        ceval_accs.append(ceval_acc)
+
+    tab = '| exp_name | model_type | dataset | ms-bench mix ratio | tuner | tuner_params | trainable params(M) | flash_attn | gradient_checkpointing | hypers | memory | train speed(samples/s) | infer speed(tokens/s) | train_loss | eval_loss | gsm8k weighted acc | arc weighted acc | ceval weighted acc |\n' \
+          '| -------- | ---------- | ------- | -------------------| ----- | ------------ | ------------------- | -----------| ---------------------- | ------ | ------ | ---------------------- | --------------------- | ---------- | --------- | ------------------ | ---------------- | ------------------ |\n' # noqa
+    min_best_metric = 999.
+    min_train_loss = 999.
+    if outputs:
+        min_best_metric = min([output.best_metric or 999. for output in outputs])
+        min_train_loss = min([output.train_loss or 999. for output in outputs])
+
+    max_gsm8k = 0.0
+    if gsm8k_accs:
+        max_gsm8k = max([gsm8k or 0. for gsm8k in gsm8k_accs])
+
+    max_arc = 0.0
+    if arc_accs:
+        max_arc = max([arc or 0. for arc in arc_accs])
+
+    max_ceval = 0.0
+    if ceval_accs:
+        max_ceval = max([ceval or 0. for ceval in ceval_accs])
+
+    for output, gsm8k_acc, arc_acc, ceval_acc in zip(outputs, gsm8k_accs, arc_accs, ceval_accs):
+        use_flash_attn = output.args.get('use_flash_attn', '')
+        use_gc = output.args.get('gradient_checkpointing', '')
+        memory = output.memory
+        train_speed = output.train_speed
+        infer_speed = output.infer_speed
+
+        is_best_metric = np.isclose(min_best_metric, output.best_metric or 999.0)
+        is_best_loss = np.isclose(min_train_loss, output.train_loss or 999.0)
+        is_best_gsm8k = np.isclose(max_gsm8k, gsm8k_acc or 0.0)
+        is_best_arc = np.isclose(max_arc, arc_acc or 0.0)
+        is_best_ceval = np.isclose(max_ceval, ceval_acc or 0.0)
+
+        if not is_best_metric:
+            best_metric = '' if not output.best_metric else f'{output.best_metric:.2f}'
+        else:
+            best_metric = '' if not output.best_metric else f'**{output.best_metric:.2f}**'
+
+        if not is_best_loss:
+            train_loss = '' if not output.train_loss else f'{output.train_loss:.2f}'
+        else:
+            train_loss = '' if not output.train_loss else f'**{output.train_loss:.2f}**'
+
+        if not is_best_gsm8k:
+            gsm8k_acc = '' if not gsm8k_acc else f'{gsm8k_acc:.3f}'
+        else:
+            gsm8k_acc = '' if not gsm8k_acc else f'**{gsm8k_acc:.3f}**'
+
+        if not is_best_arc:
+            arc_acc = '' if not arc_acc else f'{arc_acc:.3f}'
+        else:
+            arc_acc = '' if not arc_acc else f'**{arc_acc:.3f}**'
+
+        if not is_best_ceval:
+            ceval_acc = '' if not ceval_acc else f'{ceval_acc:.3f}'
+        else:
+            ceval_acc = '' if not ceval_acc else f'**{ceval_acc:.3f}**'
+
+        line = f'|{output.name}|' \
+               f'{output.args["model_type"]}|' \
+               f'{output.args.get("dataset")}|' \
+               f'{output.args.get("train_dataset_mix_ratio", 0.)}|' \
+               f'{output.args.get("sft_type")}|' \
+               f'{output.tuner_hyper_params}|' \
+               f'{output.num_trainable_parameters}({output.trainable_parameters_percentage})|' \
+               f'{use_flash_attn}|' \
+               f'{use_gc}|' \
+               f'{output.hyper_parameters}|' \
+               f'{memory}|' \
+               f'{train_speed}|' \
+               f'{infer_speed}|' \
+               f'{best_metric}|' \
+               f'{train_loss}|' \
+               f'{gsm8k_acc}|' \
+               f'{arc_acc}|' \
+               f'{ceval_acc}|\n'
+        tab += line
+    return tab
+
+
+def generate_export_report(outputs: List[ModelOutput]):
+    tab = '| exp_name | model_type | calibration dataset | quantization method | quantization bits | infer speed(tokens/s) | gsm8k weighted acc | arc weighted acc | ceval weighted acc |\n' \
+          '| -------- | ---------- | ------------------- | ------------------- | ----------------- | --------------------- | ------------------ | ---------------- | ------------------ |\n' # noqa
+
+    gsm8k_accs = []
+    arc_accs = []
+    ceval_accs = []
+    for output in outputs:
+        gsm8k_acc = None
+        arc_acc = None
+        ceval_acc = None
+        for report in (output.reports or []):
+            if report['name'] == 'gsm8k':
+                gsm8k_acc = report['score']
+            if report['name'] == 'arc':
+                arc_acc = report['score']
+            if report['name'] == 'ceval':
+                ceval_acc = report['score']
+        gsm8k_accs.append(gsm8k_acc)
+        arc_accs.append(arc_acc)
+        ceval_accs.append(ceval_acc)
+
+    max_gsm8k = 0.0
+    if gsm8k_accs:
+        max_gsm8k = max([gsm8k or 0. for gsm8k in gsm8k_accs])
+
+    max_arc = 0.0
+    if arc_accs:
+        max_arc = max([arc or 0. for arc in arc_accs])
+
+    max_ceval = 0.0
+    if ceval_accs:
+        max_ceval = max([ceval or 0. for ceval in ceval_accs])
+
+    for output, gsm8k_acc, arc_acc, ceval_acc in zip(outputs, gsm8k_accs, arc_accs, ceval_accs):
+        infer_speed = output.infer_speed
+        is_best_gsm8k = np.isclose(max_gsm8k, gsm8k_acc or 0.0)
+        is_best_arc = np.isclose(max_arc, arc_acc or 0.0)
+        is_best_ceval = np.isclose(max_ceval, ceval_acc or 0.0)
+
+        if not is_best_gsm8k:
+            gsm8k_acc = '' if not gsm8k_acc else f'{gsm8k_acc:.3f}'
+        else:
+            gsm8k_acc = '' if not gsm8k_acc else f'**{gsm8k_acc:.3f}**'
+
+        if not is_best_arc:
+            arc_acc = '' if not arc_acc else f'{arc_acc:.3f}'
+        else:
+            arc_acc = '' if not arc_acc else f'**{arc_acc:.3f}**'
+
+        if not is_best_ceval:
+            ceval_acc = '' if not ceval_acc else f'{ceval_acc:.3f}'
+        else:
+            ceval_acc = '' if not ceval_acc else f'**{ceval_acc:.3f}**'
+
+        if output.train_dataset_info:
+            dataset_info = f'{output.args["dataset"]}/{output.train_dataset_info}'
+        else:
+            dataset_info = f'{output.args["dataset"]}'
+        line = f'|{output.name}|' \
+               f'{output.args["model_type"]}|' \
+               f'{dataset_info}|' \
+               f'{output.args["quant_method"]}|' \
+               f'{output.args["quant_bits"]}|' \
+               f'{infer_speed}|' \
+               f'{gsm8k_acc}|' \
+               f'{arc_acc}|' \
+               f'{ceval_acc}|\n'
+        tab += line
+    return tab
+
+
+def parse_output(file):
+    with open(file, 'r', encoding='utf-8') as f:
+        content = json.load(f)
+
+    name = content['name']
+    group = content['group']
+    cmd = content['cmd']
+    requirements = content['requirements']
+    args = content['args']
+    create_time = float(content.get('create_time') or 0)
+    content = content['record']
+    if cmd == 'export':
+        best_model_checkpoint = content['best_model_checkpoint']
+        eval_tokens = 0
+        eval_time = 0.0
+        eval_result = None
+        if 'eval_result' in content:
+            eval_result = content['eval_result']
+            eval_tokens = eval_result['generation_info']['tokens']
+            eval_time = eval_result['generation_info']['time']
+            eval_result = eval_result['report']
+        return ModelOutput(
+            group=group,
+            name=name,
+            cmd=cmd,
+            requirements=requirements,
+            args=args,
+            best_model_checkpoint=best_model_checkpoint,
+            eval_time=eval_time,
+            eval_tokens=eval_tokens,
+            reports=eval_result,
+        )
+    else:
+        memory = None
+        train_time = None
+        train_samples = None
+        train_samples_per_second = None
+        last_model_checkpoint = None
+        best_model_checkpoint = None
+        best_metric = None
+        global_step = None
+        train_dataset_info = None
+        val_dataset_info = None
+        num_trainable_parameters = None
+        num_buffers = None
+        trainable_parameters_percentage = None
+        num_total_parameters = None
+        train_loss = None
+        if 'memory' in content:
+            memory = content['memory']
+            memory = '/'.join(memory.values())
+        if 'train_time' in content:
+            train_time = content['train_time']['train_runtime']
+            train_samples = content['train_time']['n_train_samples']
+            train_samples_per_second = content['train_time']['train_samples_per_second']
+        if 'last_model_checkpoint' in content:
+            last_model_checkpoint = content['last_model_checkpoint']
+        if 'best_model_checkpoint' in content:
+            best_model_checkpoint = content['best_model_checkpoint']
+        if 'best_metric' in content:
+            best_metric = content['best_metric']
+        if 'log_history' in content:
+            train_loss = content['log_history'][-1]['train_loss']
+        if 'global_step' in content:
+            global_step = content['global_step']
+        if 'dataset_info' in content:
+            train_dataset_info = content['dataset_info'].get('train_dataset')
+            val_dataset_info = content['dataset_info'].get('val_dataset')
+        if 'model_info' in content:
+            # model_info like: SwiftModel: 6758.4041M Params (19.9885M Trainable [0.2958%]), 16.7793M Buffers.
+            str_dict = split_str_parts_by(content['model_info'], [
+                'SwiftModel:', 'CausalLM:', 'Seq2SeqLM:', 'LMHeadModel:', 'M Params (', 'M Trainable [', ']), ',
+                'M Buffers.'
+            ])
+            str_dict = {c['key']: c['content'] for c in str_dict}
+            if 'SwiftModel:' in str_dict:
+                num_total_parameters = float(str_dict['SwiftModel:'])
+            elif 'CausalLM:' in str_dict:
+                num_total_parameters = float(str_dict['CausalLM:'])
+            elif 'Seq2SeqLM:' in str_dict:
+                num_total_parameters = float(str_dict['Seq2SeqLM:'])
+            elif 'LMHeadModel:' in str_dict:
+                num_total_parameters = float(str_dict['LMHeadModel:'])
+            num_trainable_parameters = float(str_dict['M Params ('])
+            num_buffers = float(str_dict[']), '])
+            trainable_parameters_percentage = str_dict['M Trainable [']
+
+        eval_tokens = 0
+        eval_time = 0.0
+        eval_result = None
+        if 'eval_result' in content:
+            eval_result = content['eval_result']
+            eval_tokens = eval_result['generation_info']['tokens']
+            eval_time = eval_result['generation_info']['time']
+            eval_result = eval_result['report']
+
+        return ModelOutput(
+            group=group,
+            name=name,
+            cmd=cmd,
+            requirements=requirements,
+            args=args,
+            memory=memory,
+            train_time=train_time,
+            train_samples=train_samples,
+            train_samples_per_second=train_samples_per_second,
+            last_model_checkpoint=last_model_checkpoint,
+            best_model_checkpoint=best_model_checkpoint,
+            best_metric=best_metric,
+            global_step=global_step,
+            train_dataset_info=train_dataset_info,
+            val_dataset_info=val_dataset_info,
+            train_create_time=create_time,
+            num_total_parameters=num_total_parameters,
+            num_trainable_parameters=num_trainable_parameters,
+            num_buffers=num_buffers,
+            trainable_parameters_percentage=trainable_parameters_percentage,
+            eval_time=eval_time,
+            eval_tokens=eval_tokens,
+            reports=eval_result,
+            train_loss=train_loss,
+        )
+
+
+def generate_reports():
+    outputs = []
+    for dirs, _, files in os.walk('./experiment'):
+        for file in files:
+            abs_file = os.path.join(dirs, file)
+            if not abs_file.endswith('.json') or 'ipynb' in abs_file:
+                continue
+
+            outputs.append(parse_output(abs_file))
+
+    all_groups = set([output.group for output in outputs])
+    for group in all_groups:
+        group_outputs = [output for output in outputs if output.group == group]
+        print(f'=================Printing the sft cmd result of exp {group}==================\n\n')
+        print(generate_sft_report([output for output in group_outputs if output.cmd in ('sft', 'eval')]))
+        # print(f'=================Printing the dpo result of exp {group}==================')
+        # print(generate_dpo_report([output for output in outputs if output.cmd == 'dpo']))
+        print(f'=================Printing the export cmd result of exp {group}==================\n\n')
+        print(generate_export_report([output for output in group_outputs if output.cmd == 'export']))
+        print('=================Printing done==================\n\n')
+
+
+if __name__ == '__main__':
+    generate_reports()

ms-swift/scripts/utils/run_dataset_info.py

@@ -0,0 +1,106 @@
+import os
+import re
+
+import numpy as np
+
+from swift.llm import DATASET_MAPPING, EncodePreprocessor, get_model_tokenizer, get_template, load_dataset
+from swift.utils import stat_array
+
+os.environ['HF_ENDPOINT'] = 'https://hf-mirror.com'
+
+
+def get_cache_mapping(fpath):
+    with open(fpath, 'r', encoding='utf-8') as f:
+        text = f.read()
+    idx = text.find('| Dataset ID |')
+    text = text[idx:]
+    text_list = text.split('\n')[2:]
+    cache_mapping = {}  # dataset_id -> (dataset_size, stat)
+    for text in text_list:
+        if not text:
+            continue
+        items = text.split('|')
+        key = items[1] if items[1] != '-' else items[6]
+        key = re.search(r'\[(.+?)\]', key).group(1)
+        stat = items[3:5]
+        if stat[0] == '-':
+            stat = ('huge dataset', '-')
+        cache_mapping[key] = stat
+    return cache_mapping
+
+
+def get_dataset_id(key):
+    for dataset_id in key:
+        if dataset_id is not None:
+            break
+    return dataset_id
+
+
+def run_dataset(key, template, cache_mapping):
+    ms_id, hf_id, _ = key
+    dataset_meta = DATASET_MAPPING[key]
+    tags = ', '.join(tag for tag in dataset_meta.tags) or '-'
+    dataset_id = ms_id or hf_id
+    use_hf = ms_id is None
+    if ms_id is not None:
+        ms_id = f'[{ms_id}](https://modelscope.cn/datasets/{ms_id})'
+    else:
+        ms_id = '-'
+    if hf_id is not None:
+        hf_id = f'[{hf_id}](https://huggingface.co/datasets/{hf_id})'
+    else:
+        hf_id = '-'
+    subsets = '<br>'.join(subset.name for subset in dataset_meta.subsets)
+
+    if dataset_meta.huge_dataset:
+        dataset_size = 'huge dataset'
+        stat_str = '-'
+    elif dataset_id in cache_mapping:
+        dataset_size, stat_str = cache_mapping[dataset_id]
+    else:
+        num_proc = 4
+        dataset, _ = load_dataset(f'{dataset_id}:all', strict=False, num_proc=num_proc, use_hf=use_hf)
+        dataset_size = len(dataset)
+        random_state = np.random.RandomState(42)
+        idx_list = random_state.choice(dataset_size, size=min(dataset_size, 100000), replace=False)
+        encoded_dataset = EncodePreprocessor(template)(dataset.select(idx_list), num_proc=num_proc)
+
+        input_ids = encoded_dataset['input_ids']
+        token_len = [len(tokens) for tokens in input_ids]
+        stat = stat_array(token_len)[0]
+        stat_str = f"{stat['mean']:.1f}±{stat['std']:.1f}, min={stat['min']}, max={stat['max']}"
+
+    return f'|{ms_id}|{subsets}|{dataset_size}|{stat_str}|{tags}|{hf_id}|'
+
+
+def write_dataset_info() -> None:
+    fpaths = ['docs/source/Instruction/支持的模型和数据集.md', 'docs/source_en/Instruction/Supported-models-and-datasets.md']
+    cache_mapping = get_cache_mapping(fpaths[0])
+    res_text_list = []
+    res_text_list.append('| Dataset ID | Subset Name | Dataset Size | Statistic (token) | Tags | HF Dataset ID |')
+    res_text_list.append('| ---------- | ----------- | -------------| ------------------| ---- | ------------- |')
+
+    all_keys = list(DATASET_MAPPING.keys())
+    all_keys = sorted(all_keys, key=lambda x: get_dataset_id(x))
+    _, tokenizer = get_model_tokenizer('Qwen/Qwen2.5-7B-Instruct', load_model=False)
+    template = get_template(tokenizer.model_meta.template, tokenizer)
+    try:
+        for i, key in enumerate(all_keys):
+            res = run_dataset(key, template, cache_mapping)
+            res_text_list.append(res)
+            print(res)
+    finally:
+        for fpath in fpaths:
+            with open(fpath, 'r', encoding='utf-8') as f:
+                text = f.read()
+            idx = text.find('| Dataset ID |')
+
+            new_text = '\n'.join(res_text_list)
+            text = text[:idx] + new_text + '\n'
+            with open(fpath, 'w', encoding='utf-8') as f:
+                f.write(text)
+    print(f'数据集总数: {len(all_keys)}')
+
+
+if __name__ == '__main__':
+    write_dataset_info()

ms-swift/scripts/utils/run_template.py

@@ -0,0 +1,8 @@
+from swift.llm import TemplateType
+
+if __name__ == '__main__':
+    template_name_list = TemplateType.get_template_name_list()
+    tn_gen = ', '.join([tn for tn in template_name_list if 'generation' in tn])
+    tn_chat = ', '.join([tn for tn in template_name_list if 'generation' not in tn])
+    print(f'Text Generation: {tn_gen}')
+    print(f'Chat: {tn_chat}')

ms-swift/swift/__init__.py

@@ -0,0 +1,55 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
+
+from .utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .version import __version__, __release_datetime__
+    from .tuners import (Adapter, AdapterConfig, AdapterModule, SwiftModel, LoRA, LoRAConfig, SWIFT_MAPPING,
+                         AdaLoraConfig, LoftQConfig, LoHaConfig, LoKrConfig, LoraConfig, OFTConfig, PeftConfig,
+                         PeftModel, PeftModelForCausalLM, ResTuningConfig, SideConfig, PeftModelForSeq2SeqLM,
+                         PeftModelForSequenceClassification, PeftModelForTokenClassification, PrefixTuningConfig,
+                         PromptEncoderConfig, PromptLearningConfig, PromptTuningConfig, get_peft_config, get_peft_model,
+                         get_peft_model_state_dict, Prompt, PromptConfig, PromptModule, SwiftConfig, SwiftOutput, Swift,
+                         SwiftTuners, LongLoRAConfig, LongLoRA, LongLoRAModelType, SCETuning, SCETuningConfig)
+    from .trainers import (EvaluationStrategy, FSDPOption, HPSearchBackend, HubStrategy, IntervalStrategy,
+                           SchedulerType, ShardedDDPOption, TrainingArguments, Seq2SeqTrainingArguments, Trainer,
+                           Seq2SeqTrainer)
+    from .utils import get_logger
+else:
+    _import_structure = {
+        'version': ['__release_datetime__', '__version__'],
+        'tuners': [
+            'Adapter', 'AdapterConfig', 'AdapterModule', 'SwiftModel', 'LoRA', 'LoRAConfig', 'SWIFT_MAPPING',
+            'LoraConfig', 'AdaLoraConfig', 'LoftQConfig', 'LoHaConfig', 'LoKrConfig', 'OFTConfig', 'PeftConfig',
+            'ResTuningConfig', 'SideConfig', 'PeftModel', 'PeftModelForCausalLM', 'PeftModelForSeq2SeqLM',
+            'PeftModelForSequenceClassification', 'PeftModelForTokenClassification', 'PrefixTuningConfig',
+            'PromptEncoderConfig', 'PromptLearningConfig', 'PromptTuningConfig', 'get_peft_config', 'get_peft_model',
+            'get_peft_model_state_dict', 'Prompt', 'PromptConfig', 'PromptModule', 'SwiftConfig', 'SwiftOutput',
+            'Swift', 'SwiftTuners', 'LongLoRAConfig', 'LongLoRA', 'LongLoRAModelType', 'SCETuning', 'SCETuningConfig'
+        ],
+        'trainers': [
+            'EvaluationStrategy',
+            'FSDPOption',
+            'HPSearchBackend',
+            'HubStrategy',
+            'IntervalStrategy',
+            'SchedulerType',
+            'ShardedDDPOption',
+            'TrainingArguments',
+            'Seq2SeqTrainingArguments',
+            'Trainer',
+            'Seq2SeqTrainer',
+        ],
+        'utils': ['get_logger']
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

ms-swift/swift/cli/_megatron/pt.py

@@ -0,0 +1,4 @@
+from swift.megatron import megatron_pt_main
+
+if __name__ == '__main__':
+    megatron_pt_main()

ms-swift/swift/cli/_megatron/sft.py

@@ -0,0 +1,4 @@
+from swift.megatron import megatron_sft_main
+
+if __name__ == '__main__':
+    megatron_sft_main()

ms-swift/swift/cli/app.py

@@ -0,0 +1,4 @@
+from swift.llm import app_main
+
+if __name__ == '__main__':
+    app_main()

ms-swift/swift/cli/eval.py

@@ -0,0 +1,5 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from swift.llm import eval_main
+
+if __name__ == '__main__':
+    eval_main()

ms-swift/swift/cli/export.py

@@ -0,0 +1,5 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from swift.llm import export_main
+
+if __name__ == '__main__':
+    export_main()

ms-swift/swift/cli/main.py

@@ -0,0 +1,76 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import importlib.util
+import os
+import subprocess
+import sys
+from typing import Dict, List, Optional
+
+from swift.utils import get_logger
+
+logger = get_logger()
+
+ROUTE_MAPPING: Dict[str, str] = {
+    'pt': 'swift.cli.pt',
+    'sft': 'swift.cli.sft',
+    'infer': 'swift.cli.infer',
+    'merge-lora': 'swift.cli.merge_lora',
+    'web-ui': 'swift.cli.web_ui',
+    'deploy': 'swift.cli.deploy',
+    'rollout': 'swift.cli.rollout',
+    'rlhf': 'swift.cli.rlhf',
+    'sample': 'swift.cli.sample',
+    'export': 'swift.cli.export',
+    'eval': 'swift.cli.eval',
+    'app': 'swift.cli.app',
+}
+
+
+def use_torchrun() -> bool:
+    nproc_per_node = os.getenv('NPROC_PER_NODE')
+    nnodes = os.getenv('NNODES')
+    if nproc_per_node is None and nnodes is None:
+        return False
+    return True
+
+
+def get_torchrun_args() -> Optional[List[str]]:
+    if not use_torchrun():
+        return
+    torchrun_args = []
+    for env_key in ['NPROC_PER_NODE', 'MASTER_PORT', 'NNODES', 'NODE_RANK', 'MASTER_ADDR']:
+        env_val = os.getenv(env_key)
+        if env_val is None:
+            continue
+        torchrun_args += [f'--{env_key.lower()}', env_val]
+    return torchrun_args
+
+
+def _compat_web_ui(argv):
+    # [compat]
+    method_name = argv[0]
+    if method_name in {'web-ui', 'web_ui'} and ('--model' in argv or '--adapters' in argv or '--ckpt_dir' in argv):
+        argv[0] = 'app'
+        logger.warning('Please use `swift app`.')
+
+
+def cli_main(route_mapping: Optional[Dict[str, str]] = None) -> None:
+    route_mapping = route_mapping or ROUTE_MAPPING
+    argv = sys.argv[1:]
+    _compat_web_ui(argv)
+    method_name = argv[0].replace('_', '-')
+    argv = argv[1:]
+    file_path = importlib.util.find_spec(route_mapping[method_name]).origin
+    torchrun_args = get_torchrun_args()
+    python_cmd = sys.executable
+    if torchrun_args is None or method_name not in {'pt', 'sft', 'rlhf', 'infer'}:
+        args = [python_cmd, file_path, *argv]
+    else:
+        args = [python_cmd, '-m', 'torch.distributed.run', *torchrun_args, file_path, *argv]
+    print(f"run sh: `{' '.join(args)}`", flush=True)
+    result = subprocess.run(args)
+    if result.returncode != 0:
+        sys.exit(result.returncode)
+
+
+if __name__ == '__main__':
+    cli_main()

ms-swift/swift/cli/pt.py

@@ -0,0 +1,5 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from swift.llm import pt_main
+
+if __name__ == '__main__':
+    pt_main()

ms-swift/swift/cli/rollout.py

@@ -0,0 +1,5 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from swift.llm import rollout_main
+
+if __name__ == '__main__':
+    rollout_main()

ms-swift/swift/llm/app/__init__.py

@@ -0,0 +1 @@
+from .app import SwiftApp, app_main

ms-swift/swift/llm/argument/app_args.py

@@ -0,0 +1,38 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass
+from typing import Literal, Optional
+
+from swift.utils import find_free_port, get_logger
+from ..model import get_matched_model_meta
+from ..template import get_template_meta
+from .deploy_args import DeployArguments
+from .webui_args import WebUIArguments
+
+logger = get_logger()
+
+
+@dataclass
+class AppArguments(WebUIArguments, DeployArguments):
+    base_url: Optional[str] = None
+    studio_title: Optional[str] = None
+    is_multimodal: Optional[bool] = None
+
+    lang: Literal['en', 'zh'] = 'en'
+    verbose: bool = False
+
+    def _init_torch_dtype(self) -> None:
+        if self.base_url:
+            self.model_meta = get_matched_model_meta(self.model)
+            return
+        super()._init_torch_dtype()
+
+    def __post_init__(self):
+        super().__post_init__()
+        self.server_port = find_free_port(self.server_port)
+        if self.model_meta:
+            if self.system is None:
+                self.system = get_template_meta(self.model_meta.template).default_system
+            if self.is_multimodal is None:
+                self.is_multimodal = self.model_meta.is_multimodal
+        if self.is_multimodal is None:
+            self.is_multimodal = False

ms-swift/swift/llm/data_loader.py

@@ -0,0 +1,105 @@
+from typing import Optional
+
+import torch
+import torch.distributed as dist
+from torch.utils.data import DataLoader
+
+
+class BatchSamplerShard:
+
+    def __init__(self, total_samples: int, batch_size: int, shuffle: bool, drop_last: bool, data_seed: Optional[int]):
+        self.total_samples = total_samples // self.world_size
+        self.batch_size = batch_size
+        self.shuffle = shuffle
+        self.drop_last = drop_last
+        self.base_seed = data_seed or 0
+        self.curr_seed = self.base_seed
+
+    @property
+    def rank(self):
+        return dist.get_rank() if dist.is_initialized() else 0
+
+    @property
+    def world_size(self):
+        return dist.get_world_size() if dist.is_initialized() else 1
+
+    def __iter__(self):
+        start_idx = self.rank * self.total_samples
+        if self.shuffle:
+            generator = torch.Generator()
+            generator.manual_seed(self.curr_seed)
+            total_idx = torch.randperm(self.total_samples * self.world_size, generator=generator).tolist()
+            total_idx = total_idx[start_idx:start_idx + self.total_samples]
+        else:
+            total_idx = list(range(start_idx, start_idx + self.total_samples))
+
+        batch = []
+        # Last batch if not complete will be dropped.
+        for idx in total_idx:
+            batch.append(idx)
+            if len(batch) == self.batch_size:
+                yield batch
+                batch = []
+        if not self.drop_last and len(batch) > 0:
+            yield batch
+        return
+
+    def set_epoch(self, epoch: int):
+        self.curr_seed = self.base_seed + epoch
+
+    def __len__(self) -> int:
+        if self.drop_last:
+            return self.total_samples // self.batch_size
+        else:
+            return (self.total_samples + self.batch_size - 1) // self.batch_size
+
+
+class DataLoaderShard(DataLoader):
+
+    def __init__(self, dataset, batch_sampler: BatchSamplerShard, **dataloader_params):
+        self.batch_sampler = batch_sampler
+        super().__init__(dataset, batch_sampler=self.batch_sampler, **dataloader_params)
+
+    def set_epoch(self, epoch: int):
+        self.batch_sampler.set_epoch(epoch)
+
+
+class DataLoaderDispatcher:
+
+    def __init__(self, base_dataloader):
+        self.base_dataloader = base_dataloader
+
+    @property
+    def rank(self):
+        return dist.get_rank(self.group) if dist.is_initialized() else 0
+
+    @property
+    def world_size(self):
+        return dist.get_world_size(self.group) if dist.is_initialized() else 1
+
+    @property
+    def group(self):
+        return dist.group.WORLD if dist.is_initialized() else 1
+
+    def _scatter_object_list(self, inputs):
+        if not dist.is_initialized():
+            return inputs[0]
+        outputs = [None]
+        global_src_rank = dist.get_global_rank(self.group, 0)
+        dist.scatter_object_list(outputs, inputs, global_src_rank, group=self.group)
+        return outputs[0]
+
+    def __iter__(self):
+        base_iter = iter(self.base_dataloader)
+        while True:
+            if self.rank == 0:
+                try:
+                    data = [next(base_iter) for _ in range(self.world_size)]
+                except StopIteration:
+                    data = [None] * self.world_size
+                data = self._scatter_object_list(data)
+            else:
+                data = self._scatter_object_list(None)
+            if data is None:
+                break
+            yield data

ms-swift/swift/llm/dataset/dataset/llm.py

@@ -0,0 +1,856 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import ast
+import re
+from functools import partial
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import json
+import numpy as np
+
+from ...template import split_str_parts_by
+from ..preprocessor import (AlpacaPreprocessor, ClsGenerationPreprocessor, ClsPreprocessor, MessagesPreprocessor,
+                            ResponsePreprocessor, RowPreprocessor, TextGenerationPreprocessor)
+from ..register import DatasetMeta, SubsetDataset, register_dataset
+
+
+class AlpacaZhPreprocessor(AlpacaPreprocessor):
+
+    @classmethod
+    def concat_inst_input(cls, instruction, input_):
+        if input_ and input_.startswith('输入：'):
+            input_ = input_[3:]
+        return super().concat_inst_input(instruction, input_)
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/alpaca-gpt4-data-zh',
+        hf_dataset_id='llm-wizard/alpaca-gpt4-data-zh',
+        preprocess_func=AlpacaZhPreprocessor(),
+        tags=['chat', 'general', '🔥'],
+    ))
+
+
+class LongAlpacaPreprocessor(AlpacaPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        response = row['response']
+        prefix_prompt = 'Answer: '
+        if response and response.startswith(prefix_prompt):
+            response = response[len(prefix_prompt):].strip()
+            row['output'] = response
+        return super().preprocess(row)
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/LongAlpaca-12k',
+        hf_dataset_id='Yukang/LongAlpaca-12k',
+        preprocess_func=LongAlpacaPreprocessor(),
+        tags=['long-sequence', 'QA'],
+    ))
+
+
+class RuozhibaPreprocessor(RowPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        title = row['title'] if row.get('title', None) is not None else row['content']
+        abs = row['abs'] if 'abs' in row else None
+        if abs and abs != title:
+            title = title + '，' + abs
+
+        pattern = r'\d+[\.,\s,\、](.+)'
+        match = re.search(pattern, title)
+        if match:
+            title = match.group(1)
+        if title:
+            return {'messages': [{'role': 'assistant', 'content': title}]}
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/ruozhiba',
+        subsets=['post-annual', 'title-good', 'title-norm'],
+        preprocess_func=RuozhibaPreprocessor(),
+        tags=['pretrain', '🔥']))
+
+
+class MathTrnPreprocessor(ResponsePreprocessor):
+
+    def preprocess(self, row):
+        query = row['query']
+        output = row['response']
+        row = {
+            'query': query,
+            'response': output,
+        }
+        return super().preprocess(row)
+
+
+register_dataset(
+    DatasetMeta(ms_dataset_id='AI-ModelScope/math-trn-format', preprocess_func=MathTrnPreprocessor(), tags=['math']))
+
+
+def _repair_ms_bench(messages: str) -> Optional[List[Dict[str, str]]]:
+    if isinstance(messages, str):
+        messages = ast.literal_eval(messages)
+    default_system = 'You are a helpful assistant.'
+    messages: List[Dict[str, str]]
+    if messages[0]['from'] == 'system' and messages[0]['value'] == default_system:
+        messages.pop(0)
+    # skip MOSS
+    for c in messages:
+        value = c['value'].lower()
+        if 'moss' in value or 'human:' in value or 'assistant:' in value or 'user:' in value:
+            return
+    return messages
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='iic/ms_bench',
+        preprocess_func=MessagesPreprocessor(repair_messages=_repair_ms_bench),
+        tags=['chat', 'general', 'multi-round', '🔥']))
+
+
+def _repair_agent_messages(messages: List[Dict[str, str]], use_mini: bool) -> Optional[List[Dict[str, str]]]:
+    if use_mini:
+        pattern = r'\d\. {"plugin_name": "(.+?)"'
+        if messages[0]['from'] != 'system':
+            return
+        system = messages[0]['value']
+        find_list = re.findall(pattern, system)
+        if len(set(find_list)) <= 1:
+            return
+    return messages
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='damo/MSAgent-Bench',
+        subsets=[
+            SubsetDataset(
+                preprocess_func=MessagesPreprocessor(repair_messages=partial(_repair_agent_messages, use_mini=False))),
+            SubsetDataset(
+                name='mini',
+                preprocess_func=MessagesPreprocessor(repair_messages=partial(_repair_agent_messages, use_mini=True)),
+                is_weak_subset=True)
+        ],
+        split=['train', 'validation'],
+        tags=['chat', 'agent', 'multi-round']))
+
+advertise_gen_prompt = """Task: Generating advertisements based on keywords.
+Keywords: {{QUERY}}
+Advertisements:"""
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='lvjianjin/AdvertiseGen',
+        hf_dataset_id='shibing624/AdvertiseGen',
+        preprocess_func=TextGenerationPreprocessor(
+            prompt=advertise_gen_prompt, columns={
+                'content': 'query',
+                'summary': 'response'
+            }),
+        tags=['text-generation', '🔥'],
+        split=['train', 'validation'],
+    ))
+
+
+class FireflyPreprocessor(ResponsePreprocessor):
+    _firefly_kind_list = {
+        'ProseGeneration', 'MRC', 'JinYongGeneration', 'TextCorrection', 'ClassicalChinese', 'BELLE', 'StoryGeneration',
+        'Couplet', 'Cot', 'Dictionary', 'Translation', 'Program', 'SentimentAnalyze', 'OpenQA', 'AncientPoem',
+        'TextMatching', 'NLI', 'Summary', 'KeywordRecognition', 'ProductDesc', 'LyricGeneration', 'Composition',
+        'MusicComment', 'NER'
+    }
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        if row['kind'] not in FireflyPreprocessor._firefly_kind_list:
+            return
+        return super().preprocess(row)
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/firefly-train-1.1M',
+        hf_dataset_id='YeungNLP/firefly-train-1.1M',
+        preprocess_func=FireflyPreprocessor(),
+        tags=['chat', 'general'],
+    ))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='modelscope/clue',
+        hf_dataset_id='clue',
+        subsets=['cmnli'],
+        preprocess_func=ClsGenerationPreprocessor(['neutral', 'entailment', 'contradiction'],
+                                                  task='Natural Language Inference',
+                                                  is_pair_seq=True),
+        tags=['text-generation', 'classification'],
+        split=['train', 'validation'],
+    ))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='DAMO_NLP/jd',
+        subsets=[
+            SubsetDataset(
+                'default',
+                'default',
+                preprocess_func=ClsGenerationPreprocessor(['negative', 'positive'],
+                                                          task='Sentiment Classification',
+                                                          is_pair_seq=False)),
+            SubsetDataset(
+                'cls',
+                'default',
+                preprocess_func=ClsPreprocessor(columns={'sentence': 'query'}),
+            ),
+        ],
+        tags=['text-generation', 'classification', '🔥'],
+        split=['train', 'validation'],
+    ))
+
+
+class SyntheticText2SqlPreprocessor(ResponsePreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        sql_prompt = row['sql_prompt']
+        sql_context = row['sql_context']
+        sql = row['sql']
+        sql_explanation = row['sql_explanation']
+        query = f'Sql Table information:\n{sql_context}\n{sql_prompt}'
+        response = f'Let\'s think step by step:\n{sql_explanation}\nSo the final sql is:\n{sql}'
+        return super().preprocess({'query': query, 'response': response})
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/synthetic_text_to_sql',
+        hf_dataset_id='gretelai/synthetic_text_to_sql',
+        preprocess_func=SyntheticText2SqlPreprocessor(),
+        tags=['nl2sql', 'en']))
+
+
+def _repair_toolbench(conversations: List[Dict[str, str]]) -> List[Dict[str, str]]:
+    assert len(conversations) == 2
+    if conversations[1]['from'] in {'caller', 'conclusion'}:
+        conversations[1]['from'] = 'assistant'
+    return conversations
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='shenweizhou/alpha-umi-toolbench-processed-v2',
+        subsets=['backbone', 'caller', 'planner', 'summarizer'],
+        preprocess_func=MessagesPreprocessor(repair_messages=_repair_toolbench),
+        tags=['chat', 'agent', '🔥'],
+        huge_dataset=True))
+
+
+class BlossomMathPreprocessor(ResponsePreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        output, answer = row['output'], row['answer']
+        return super().preprocess({'query': row['query'], 'response': f'{output}\n\nAnswer: {answer}'})
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/blossom-math-v2',
+        hf_dataset_id='Azure99/blossom-math-v2',
+        preprocess_func=BlossomMathPreprocessor(),
+        tags=['chat', 'math', '🔥']))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/sql-create-context',
+        hf_dataset_id='b-mc2/sql-create-context',
+        preprocess_func=AlpacaPreprocessor(columns={
+            'question': 'instruction',
+            'context': 'input',
+            'answer': 'output'
+        }),
+        tags=['chat', 'sql', '🔥']))
+
+
+class TigerBotLawPreprocessor(ResponsePreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        prompt = """{type}
+{title}
+"""
+        cur_prompt = prompt.format(type=row['type'], title=row['title'])
+        for i in range(1, 4):
+            chapter = row[f'chapter{i}']
+            if chapter is not None:
+                cur_prompt += f'{chapter}'
+        cur_prompt += f'{row["response"]}'
+        return super().preprocess({'response': cur_prompt})
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/tigerbot-law-plugin',
+        hf_dataset_id='TigerResearch/tigerbot-law-plugin',
+        preprocess_func=TigerBotLawPreprocessor(),
+        tags=['text-generation', 'law', 'pretrained']))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='codefuse-ai/CodeExercise-Python-27k',
+        preprocess_func=MessagesPreprocessor(columns={'chat_rounds': 'messages'}),
+        tags=['chat', 'coding', '🔥']))
+
+
+class LeetcodePythonPreprocessor(ResponsePreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        code_with_problem = row['code_with_problem']
+        idx = code_with_problem.find('```python')
+        problem = code_with_problem[:idx]
+        if problem.startswith('# '):
+            problem = problem[2:]
+        code = code_with_problem[idx:].strip()
+        explanation = row['explanation_only']
+        return super().preprocess({'query': problem, 'response': f'{code}\n\n{explanation}'})
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/leetcode-solutions-python',
+        preprocess_func=LeetcodePythonPreprocessor(),
+        tags=['chat', 'coding', '🔥']))
+
+
+class StsbPreprocessor(ResponsePreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        row = {
+            'query': row['sentence1'],
+            'response': row['sentence2'],
+            'label': row['score'],
+        }
+        return super().preprocess(row)
+
+
+class StsbGeneratePreprocessor(ResponsePreprocessor):
+    prompt = """Task: Based on the given two sentences, provide a similarity score between 0.0 and 1.0.
+Sentence 1: {text1}
+Sentence 2: {text2}
+Similarity score: """
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        return super().preprocess({
+            'query': self.prompt.format(text1=row['sentence1'], text2=row['sentence2']),
+            'response': f"{row['score']:.1f}"
+        })
+
+
+class StsbRegressionPreprocessor(StsbGeneratePreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        return super(StsbGeneratePreprocessor, self).preprocess({
+            'query':
+            self.prompt.format(text1=row['sentence1'], text2=row['sentence2']),
+            'label':
+            row['score']
+        })
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='sentence-transformers/stsb',
+        hf_dataset_id='sentence-transformers/stsb',
+        subsets=[
+            SubsetDataset('default', preprocess_func=StsbPreprocessor()),  # embedding
+            SubsetDataset('generate', preprocess_func=StsbGeneratePreprocessor()),
+            SubsetDataset('reg', preprocess_func=StsbRegressionPreprocessor()),
+        ],
+        tags=['similarity', '🔥']))
+
+
+def _repair_conversations_agent_instruct(s: str) -> List[Dict[str, Any]]:
+    s = s.replace('}\n {', '},\n {')
+    if isinstance(s, str):
+        s = ast.literal_eval(s)
+    return s
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='huangjintao/AgentInstruct_copy',
+        subsets=['alfworld', 'db', 'kg', 'mind2web', 'os', 'webshop'],
+        preprocess_func=MessagesPreprocessor(repair_messages=_repair_conversations_agent_instruct),
+        tags=['chat', 'agent', 'multi-round']))
+
+
+class MultiRoleAgentPreprocessor(RowPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        conv = row['conversations']
+        res_prompt = '\n\n【注意事项】\n1. 这是聊天室，不要发送私信给任何人\n2. 仅代表你个人说话,不要扮演其他人，只根据对话历史进行回复\n3. 长话短说，不要说太多话，不要超过50字 '
+        history_prompt = '\n\n【chat history】'
+        conv_prompt = '\n {name}:{content}'
+        query, response = '', conv[-1]['value']
+        system = conv[0]['value'] if conv[0]['from'] == 'system' else ''
+        if conv[0]['from'] == 'user':
+            query = conv[0]['value']
+        elif 'next_speakers:' not in system:
+            if '【注意事项】' not in system and system:
+                system += res_prompt
+            system += history_prompt
+            system += ''.join([conv_prompt.format(name=c['from'], content=c['value']) for c in conv[1:-1]])
+
+        if not query or not response:
+            return
+
+        return {
+            'messages': [{
+                'role': 'system',
+                'content': system
+            }, {
+                'role': 'user',
+                'content': query
+            }, {
+                'role': 'assistant',
+                'content': response
+            }],
+        }
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='iic/MSAgent-MultiRole',
+        preprocess_func=MultiRoleAgentPreprocessor(),
+        tags=['chat', 'agent', 'multi-round', 'role-play', 'multi-agent']))
+
+register_dataset(DatasetMeta(ms_dataset_id='swift/ToolBench', tags=['chat', 'agent', 'multi-round']))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='tastelikefeet/competition_math',
+        subsets=[
+            SubsetDataset(
+                name='default',
+                subset='default',
+                split=['train', 'test'],
+            ),
+        ],
+        tags=['qa', 'math']))
+
+register_dataset(DatasetMeta(ms_dataset_id='modelscope/gsm8k', subsets=['main'], split=['train'], tags=['qa', 'math']))
+
+register_dataset(
+    DatasetMeta(ms_dataset_id='modelscope/MathR', subsets=['default', 'clean'], split=['train'], tags=['qa', 'math']))
+
+register_dataset(
+    DatasetMeta(ms_dataset_id='modelscope/MathR-32B-Distill', subsets=['data'], split=['train'], tags=['qa', 'math']))
+
+
+class CoundownTaskPreprocessor(ResponsePreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        numbers = row['nums']
+        target = row.pop('response', None)
+        query = (f'Using the numbers {numbers}, create an equation that equals {target}.\n'
+                 'You can use basic arithmetic operations (+, -, *, /) and each number can only be used once.\n'
+                 'Show your work in <think> </think> tags. And return the final equation and answer '
+                 'in <answer> </answer> tags, for example <answer> (1 + 2) / 3 * 4 = 4 </answer>.')
+        row.update({'target': target, 'query': query})
+        return super().preprocess(row)
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='zouxuhong/Countdown-Tasks-3to4',
+        subsets=['default'],
+        preprocess_func=CoundownTaskPreprocessor(),
+        tags=['math']))
+
+
+class HC3Preprocessor(ResponsePreprocessor):
+    prompt = """Classification Task: Are the following responses from a human or from ChatGPT?
+Question: {question}
+Answer: {answer}
+Category: Human, ChatGPT
+Output:"""
+
+    def preprocess(self, row):
+        rows = []
+        for response in ['Human', 'ChatGPT']:
+            query = self.prompt.format(
+                question=row['query'], answer=self.random_state.choice(row[f'{response.lower()}_answers']))
+            rows.append(super().preprocess({'query': query, 'response': response}))
+        return rows
+
+
+class HC3ClsPreprocessor(HC3Preprocessor):
+
+    def preprocess(self, row):
+        rows = []
+        for i, response in enumerate(['Human', 'ChatGPT']):
+            query = self.prompt.format(
+                question=row['query'], answer=self.random_state.choice(row[f'{response.lower()}_answers']))
+            rows.append(ResponsePreprocessor.preprocess(self, {'query': query, 'label': i}))
+        return rows
+
+
+hc3_subset_names = ['baike', 'open_qa', 'nlpcc_dbqa', 'finance', 'medicine', 'law', 'psychology']
+hc3_subsets: List[SubsetDataset] = []
+for hc3_subset_name in hc3_subset_names:
+    hc3_subsets.append(
+        SubsetDataset(
+            name=hc3_subset_name,
+            subset=hc3_subset_name,
+            preprocess_func=HC3Preprocessor(),
+        ))
+    hc3_subsets.append(
+        SubsetDataset(
+            name=f'{hc3_subset_name}_cls',
+            subset=hc3_subset_name,
+            preprocess_func=HC3ClsPreprocessor(),
+        ))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='simpleai/HC3-Chinese',
+        hf_dataset_id='Hello-SimpleAI/HC3-Chinese',
+        subsets=hc3_subsets,
+        tags=['text-generation', 'classification', '🔥']))
+
+hc3_subset_names = ['finance', 'medicine']
+hc3_subsets: List[SubsetDataset] = []
+for hc3_subset_name in hc3_subset_names:
+    hc3_subsets.append(
+        SubsetDataset(
+            name=hc3_subset_name,
+            subset=hc3_subset_name,
+            preprocess_func=HC3Preprocessor(),
+        ))
+    hc3_subsets.append(
+        SubsetDataset(
+            name=f'{hc3_subset_name}_cls',
+            subset=hc3_subset_name,
+            preprocess_func=HC3ClsPreprocessor(),
+        ))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='simpleai/HC3',
+        hf_dataset_id='Hello-SimpleAI/HC3',
+        subsets=hc3_subsets,
+        preprocess_func=HC3Preprocessor(),
+        tags=['text-generation', 'classification', '🔥']))
+
+
+class DureaderPreprocessor(RowPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        prompt = """Task: Question Generation
+Context: {context}
+Answer: {answer}
+Question:"""
+        answer, context = row['text1'].split('[SEP]')
+        return {
+            'messages': [{
+                'role': 'user',
+                'content': prompt.format(context=context, answer=answer)
+            }, {
+                'role': 'assistant',
+                'content': row['text2']
+            }]
+        }
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='modelscope/DuReader_robust-QG',
+        preprocess_func=DureaderPreprocessor(),
+        split=['train', 'validation', 'test'],
+        tags=['text-generation', '🔥']))
+
+
+class HHRLHFPreprocessor(RowPreprocessor):
+
+    @staticmethod
+    def _to_messages(data):
+        messages = []
+        for query, response in zip(data[::2], data[1::2]):
+            messages.append({'role': 'user', 'content': query})
+            messages.append({'role': 'assistant', 'content': response})
+        return messages
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        chosen = row['chosen'].strip()
+        rejected = row['rejected'].strip()
+        parts_chosen = [s.strip() for s in re.split('\n\nHuman:|\n\nAssistant:|\n\nHum:', chosen)]
+        parts_rejected = [s.strip() for s in re.split('\n\nHuman:|\n\nAssistant:|\n\nHum:', rejected)]
+        if parts_chosen[0].startswith('Human:'):
+            assert parts_rejected[0].startswith('Human:')
+            parts_chosen[0] = parts_chosen[0][6:].strip()
+            parts_rejected[0] = parts_rejected[0][6:].strip()
+        row['messages'] = self._to_messages(parts_chosen)
+        row['rejected_messages'] = self._to_messages(parts_rejected)
+        return row
+
+
+# TODO meta file broken
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/hh-rlhf',
+        subsets=['helpful-base', 'helpful-online', 'helpful-rejection-sampled'],
+        preprocess_func=HHRLHFPreprocessor(),
+        split=['train', 'test'],
+        tags=['rlhf', 'dpo'],
+        huge_dataset=True))
+
+
+class XlamFunctionCallingPreprocessor(ResponsePreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        query = row['query']
+        answers = row['response']
+        if isinstance(answers, str):
+            answers = json.loads(answers)
+        answer = np.random.choice(answers)
+        name = answer['name']
+        args = json.dumps(answer['arguments'])
+        response = f'Action: {name}\nAction Input: {args}'
+        row = {'query': query, 'response': response, 'solution': response, 'tools': row['tools']}
+        return super().preprocess(row)
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='LLM-Research/xlam-function-calling-60k',
+        subsets=['dataset'],
+        preprocess_func=XlamFunctionCallingPreprocessor(),
+        tags=['agent']))
+
+
+class HHRLHFCNPreprocessor(MessagesPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        row['messages'].append(row.pop('chosen'))
+        row['rejected_response'] = row['rejected']['text']
+        return super().preprocess(row)
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/hh_rlhf_cn',
+        subsets=['hh_rlhf', 'harmless_base_cn', 'harmless_base_en', 'helpful_base_cn', 'helpful_base_en'],
+        preprocess_func=HHRLHFCNPreprocessor(columns={'context': 'messages'}, content_key='text'),
+        split=['train', 'test'],
+        tags=['rlhf', 'dpo', '🔥']))
+
+
+def repair_conversations(s: Union[str, Any]) -> Any:
+    if isinstance(s, str):
+        s = s.replace('}\n {', '},{')
+        s = s.replace('}\n{', '},{')
+        s = s.replace('}{', '},{')
+        s = s.replace('}\n  {', '},{')
+        return ast.literal_eval(s)
+    return s
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/lmsys-chat-1m',
+        hf_dataset_id='lmsys/lmsys-chat-1m',
+        preprocess_func=MessagesPreprocessor(repair_messages=repair_conversations),
+        tags=['chat', 'em']))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='hjh0119/shareAI-Llama3-DPO-zh-en-emoji',
+        hf_dataset_id='shareAI/DPO-zh-en-emoji',
+        preprocess_func=ResponsePreprocessor(columns={
+            'answer_zh': 'response',
+            'answer_en': 'rejected_response'
+        }),
+        tags=['rlhf', 'dpo']))
+
+register_dataset(
+    DatasetMeta(ms_dataset_id='AI-ModelScope/ultrafeedback-binarized-preferences-cleaned-kto', tags=['rlhf', 'kto']))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='OmniData/Zhihu-KOL-More-Than-100-Upvotes',
+        hf_dataset_id='bzb2023/Zhihu-KOL-More-Than-100-Upvotes',
+        tags=['zhihu', 'qa']))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='OmniData/Zhihu-KOL',
+        hf_dataset_id='wangrui6/Zhihu-KOL',
+        huge_dataset=True,
+        tags=['zhihu', 'qa'],
+    ))
+
+
+class GuanacoPreprocessor(RowPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        instruction = row['instruction']
+        input = row['input']
+        output = row['output']
+        history = []
+        if instruction:
+            parts = split_str_parts_by(
+                instruction, ['User:', 'User：', 'Assistant：', 'Assistant:', 'Asssistent:', 'Assistent:', 'Assistenz:'])
+            for idx, part in enumerate(parts):
+                if idx % 2 == 0:
+                    if 'user' not in part['key'].lower():
+                        return
+                    history.append([part['content'], None])
+                else:
+                    if 'assist' not in part['key'].lower() and 'asssist' not in part['key'].lower():
+                        return
+                    history[-1][-1] = part['content']
+        if input.startswith('User:'):
+            input = input[len('User:'):].strip()
+        if any([not h[0] or not h[1] for h in history]):
+            return
+
+        messages = []
+        for h in history:
+            messages.append({'role': 'user', 'content': h[0]})
+            messages.append({'role': 'assistant', 'content': h[1]})
+        messages.append({'role': 'user', 'content': input})
+        messages.append({'role': 'assistant', 'content': output})
+        return {
+            'messages': messages,
+        }
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/GuanacoDataset',
+        hf_dataset_id='JosephusCheung/GuanacoDataset',
+        preprocess_func=GuanacoPreprocessor(),
+        tags=['chat', 'zh']))
+
+
+class FunctionCallChatmlPreprocessor(MessagesPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        res = super().preprocess(row)
+
+        if res['function_description']:
+            res['tools'] = res['function_description'].split('\n\n')
+        messages = res['messages']
+        if messages[0]['role'] == 'system':
+            messages.pop(0)
+        return res
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/function-calling-chatml',
+        hf_dataset_id='Locutusque/function-calling-chatml',
+        preprocess_func=FunctionCallChatmlPreprocessor(),
+        tags=['agent', 'en', 'sft', '🔥']))
+
+
+class Dolly15kPreprocessor(RowPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        instruction = row['instruction']
+        context = row['context']
+        response = row['response']
+        query = ''
+        if context:
+            query = 'Here gives some useful information:\n'
+            query += context
+            query += '\n'
+        query += instruction
+        return {
+            'messages': [{
+                'role': 'user',
+                'content': query
+            }, {
+                'role': 'assistant',
+                'content': response
+            }],
+        }
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/databricks-dolly-15k',
+        hf_dataset_id='databricks/databricks-dolly-15k',
+        preprocess_func=Dolly15kPreprocessor(),
+        tags=['multi-task', 'en', 'quality']))
+
+
+class OrpoDPOMix40kPreprocessor(MessagesPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        if row['source'] == 'toxic-dpo-v0.2':
+            return
+        return super().preprocess(row)
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='AI-ModelScope/orpo-dpo-mix-40k',
+        hf_dataset_id='mlabonne/orpo-dpo-mix-40k',
+        preprocess_func=OrpoDPOMix40kPreprocessor(columns={
+            'chosen': 'messages',
+            'rejected': 'rejected_messages'
+        }),
+        tags=['dpo', 'orpo', 'en', 'quality']))
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='swift/sharegpt',
+        subsets=['common-zh', 'unknow-zh', 'common-en'],
+        tags=['chat', 'general', 'multi-round']))
+
+
+class SelfCognitionPreprocessor(ResponsePreprocessor):
+    name: Optional[Tuple[str, str]] = None
+    author: Optional[Tuple[str, str]] = None
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        for key in ['name', 'author']:
+            val = getattr(self, key)
+            if val is None:
+                continue
+            val = val[0] if row['tag'] == 'zh' else val[1]
+            if val is None:
+                continue
+            placeholder = '{{' + key.upper() + '}}'
+            row['query'] = row['query'].replace(placeholder, val)
+            row['response'] = row['response'].replace(placeholder, val)
+        return super().preprocess(row)
+
+
+class Qwen3SelfCognitionPreprocessor(SelfCognitionPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        row['query'] = row['query'] + ' /no_think'
+        row['response'] = '<think>\n\n</think>\n\n' + row['response']
+        return super().preprocess(row)
+
+
+class EmptyThinkSelfCognitionPreprocessor(SelfCognitionPreprocessor):
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        row['response'] = '<think>\n\n</think>\n\n' + row['response']
+        return super().preprocess(row)
+
+
+register_dataset(
+    DatasetMeta(
+        ms_dataset_id='swift/self-cognition',
+        hf_dataset_id='modelscope/self-cognition',
+        subsets=[
+            SubsetDataset(preprocess_func=SelfCognitionPreprocessor()),
+            SubsetDataset('qwen3', preprocess_func=Qwen3SelfCognitionPreprocessor()),
+            SubsetDataset('empty_think', preprocess_func=EmptyThinkSelfCognitionPreprocessor()),
+        ],
+        tags=['chat', 'self-cognition', '🔥']))

ms-swift/swift/llm/dataset/register.py

@@ -0,0 +1,177 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from copy import deepcopy
+from dataclasses import dataclass, field
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
+
+import json
+
+from swift.utils import get_logger, use_hf_hub
+from .preprocessor import DATASET_TYPE, AutoPreprocessor, MessagesPreprocessor
+
+PreprocessFunc = Callable[..., DATASET_TYPE]
+LoadFunction = Callable[..., DATASET_TYPE]
+logger = get_logger()
+
+
+@dataclass
+class SubsetDataset:
+    # `Name` is used for matching subsets of the dataset, and `subset` refers to the subset_name on the hub.
+    name: Optional[str] = None
+    # If set to None, then subset is set to subset_name.
+    subset: str = 'default'
+
+    # Higher priority. If set to None, the attributes of the DatasetMeta will be used.
+    split: Optional[List[str]] = None
+    preprocess_func: Optional[PreprocessFunc] = None
+
+    # If the dataset specifies "all," weak subsets will be skipped.
+    is_weak_subset: bool = False
+
+    def __post_init__(self):
+        if self.name is None:
+            self.name = self.subset
+
+    def set_default(self, dataset_meta: 'DatasetMeta') -> 'SubsetDataset':
+        subset_dataset = deepcopy(self)
+        for k in ['split', 'preprocess_func']:
+            v = getattr(subset_dataset, k)
+            if v is None:
+                setattr(subset_dataset, k, deepcopy(getattr(dataset_meta, k)))
+        return subset_dataset
+
+
+@dataclass
+class DatasetMeta:
+    ms_dataset_id: Optional[str] = None
+    hf_dataset_id: Optional[str] = None
+    dataset_path: Optional[str] = None
+    dataset_name: Optional[str] = None
+    ms_revision: Optional[str] = None
+    hf_revision: Optional[str] = None
+
+    subsets: List[Union[SubsetDataset, str]] = field(default_factory=lambda: ['default'])
+    # Applicable to all subsets.
+    split: List[str] = field(default_factory=lambda: ['train'])
+    # First perform column mapping, then proceed with the preprocess_func.
+    preprocess_func: PreprocessFunc = field(default_factory=lambda: AutoPreprocessor())
+    load_function: Optional[LoadFunction] = None
+
+    tags: List[str] = field(default_factory=list)
+    help: Optional[str] = None
+    huge_dataset: bool = False
+
+    def __post_init__(self):
+        from .loader import DatasetLoader
+        if self.load_function is None:
+            self.load_function = DatasetLoader.load
+        for i, subset in enumerate(self.subsets):
+            if isinstance(subset, str):
+                self.subsets[i] = SubsetDataset(subset=subset)
+
+
+DATASET_MAPPING: Dict[Tuple[str, str, str], DatasetMeta] = {}
+
+
+def get_dataset_list():
+    datasets = []
+    for key in DATASET_MAPPING:
+        if use_hf_hub():
+            if key[1]:
+                datasets.append(key[1])
+        else:
+            if key[0]:
+                datasets.append(key[0])
+    return datasets
+
+
+def register_dataset(dataset_meta: DatasetMeta, *, exist_ok: bool = False) -> None:
+    """Register dataset
+
+    Args:
+        dataset_meta: The `DatasetMeta` info of the dataset.
+        exist_ok: If the dataset id exists, raise error or update it.
+    """
+    if dataset_meta.dataset_name:
+        dataset_name = dataset_meta.dataset_name
+    else:
+        dataset_name = dataset_meta.ms_dataset_id, dataset_meta.hf_dataset_id, dataset_meta.dataset_path
+    if not exist_ok and dataset_name in DATASET_MAPPING:
+        raise ValueError(f'The `{dataset_name}` has already been registered in the DATASET_MAPPING.')
+
+    DATASET_MAPPING[dataset_name] = dataset_meta
+
+
+def _preprocess_d_info(d_info: Dict[str, Any], *, base_dir: Optional[str] = None) -> Dict[str, Any]:
+    d_info = deepcopy(d_info)
+
+    columns = None
+    if 'columns' in d_info:
+        columns = d_info.pop('columns')
+
+    if 'messages' in d_info:
+        d_info['preprocess_func'] = MessagesPreprocessor(**d_info.pop('messages'), columns=columns)
+    else:
+        d_info['preprocess_func'] = AutoPreprocessor(columns=columns)
+
+    if 'dataset_path' in d_info:
+        dataset_path = d_info.pop('dataset_path')
+        if base_dir is not None and not os.path.isabs(dataset_path):
+            dataset_path = os.path.join(base_dir, dataset_path)
+        dataset_path = os.path.abspath(os.path.expanduser(dataset_path))
+
+        d_info['dataset_path'] = dataset_path
+
+    if 'subsets' in d_info:
+        subsets = d_info.pop('subsets')
+        for i, subset in enumerate(subsets):
+            if isinstance(subset, dict):
+                subsets[i] = SubsetDataset(**_preprocess_d_info(subset))
+        d_info['subsets'] = subsets
+    return d_info
+
+
+def _register_d_info(d_info: Dict[str, Any], *, base_dir: Optional[str] = None) -> DatasetMeta:
+    """Register a single dataset to dataset mapping
+
+    Args:
+        d_info: The dataset info
+    """
+    d_info = _preprocess_d_info(d_info, base_dir=base_dir)
+    dataset_meta = DatasetMeta(**d_info)
+    register_dataset(dataset_meta)
+    return dataset_meta
+
+
+def register_dataset_info(dataset_info: Union[str, List[str], None] = None) -> List[DatasetMeta]:
+    """Register dataset from the `dataset_info.json` or a custom dataset info file
+    This is used to deal with the datasets defined in the json info file.
+
+    Args:
+        dataset_info: The dataset info path
+    """
+    # dataset_info_path: path, json or None
+    if dataset_info is None:
+        dataset_info = os.path.join(os.path.dirname(__file__), 'data', 'dataset_info.json')
+    assert isinstance(dataset_info, (str, list))
+    base_dir = None
+    log_msg = None
+    if isinstance(dataset_info, str):
+        dataset_path = os.path.abspath(os.path.expanduser(dataset_info))
+        if os.path.isfile(dataset_path):
+            log_msg = dataset_path
+            base_dir = os.path.dirname(dataset_path)
+            with open(dataset_path, 'r', encoding='utf-8') as f:
+                dataset_info = json.load(f)
+        else:
+            dataset_info = json.loads(dataset_info)  # json
+    if len(dataset_info) == 0:
+        return []
+    res = []
+    for d_info in dataset_info:
+        res.append(_register_d_info(d_info, base_dir=base_dir))
+
+    if log_msg is None:
+        log_msg = dataset_info if len(dataset_info) < 5 else list(dataset_info.keys())
+    logger.info(f'Successfully registered `{log_msg}`.')
+    return res

ms-swift/swift/llm/ds_config/zero0.json

@@ -0,0 +1,31 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+
+    "bf16": {
+        "enabled": "auto"
+    },
+
+    "zero_optimization": {
+        "stage": 0,
+        "allgather_partitions": true,
+        "allgather_bucket_size": 2e8,
+        "overlap_comm": false,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 2e8,
+        "contiguous_gradients": true
+    },
+
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 2000,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}