Add files using upload-large-folder tool

ms-swift/silence_overlaps/700/test/.ipynb_checkpoints/overlap5s_segments_test-checkpoint.json

@@ -0,0 +1,27 @@
+[
+  {
+    "key": "SODA_PROCESSED--train--123906",
+    "audio_url": "/root/autodl-tmp/output_overlapslong/newoverlapjson/overlap5s700/SODA_PROCESSED--train--123906.wav",
+    "model_output": "Multiple speakers talk simultaneously from 00:03-00:09"
+  },
+  {
+    "key": "SODA_PROCESSED--train--1112763",
+    "audio_url": "/root/autodl-tmp/output_overlapslong/newoverlapjson/overlap5s700/SODA_PROCESSED--train--1112763.wav",
+    "model_output": "Multiple speakers talk simultaneously from 00:09-00:15"
+  },
+  {
+    "key": "SODA_PROCESSED--train--790538",
+    "audio_url": "/root/autodl-tmp/output_overlapslong/newoverlapjson/overlap5s700/SODA_PROCESSED--train--790538.wav",
+    "model_output": "Multiple speakers talk simultaneously from 00:15-00:19"
+  },
+  {
+    "key": "SODA_PROCESSED--train--822773",
+    "audio_url": "/root/autodl-tmp/output_overlapslong/newoverlapjson/overlap5s700/SODA_PROCESSED--train--822773.wav",
+    "model_output": "Multiple speakers talk simultaneously from 00:14-00:19"
+  },
+  {
+    "key": "SODA_PROCESSED--train--424960",
+    "audio_url": "/root/autodl-tmp/output_overlapslong/newoverlapjson/overlap5s700/SODA_PROCESSED--train--424960.wav",
+    "model_output": "Multiple speakers talk simultaneously from 00:29-00:33"
+  }
+]

ms-swift/swift/llm/model/model/glm.py

@@ -0,0 +1,371 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, Type
+
+import torch
+import transformers
+from packaging import version
+from transformers import AutoConfig, AutoTokenizer, PreTrainedTokenizerBase
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from transformers.models.auto.tokenization_auto import get_tokenizer_config
+
+from swift.llm import TemplateType
+from swift.utils import get_device_count, get_dist_setting, get_logger
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..patcher import patch_output_to_input_device
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import AttnImpl, ModelInfo, safe_snapshot_download
+
+logger = get_logger()
+
+
+def remove_property(tokenizer_cls: Type[PreTrainedTokenizerBase], tokenizer_config: Dict[str, Any]) -> None:
+    for k, v in tokenizer_cls.__dict__.items():
+        if k.endswith('_token') and isinstance(v, property) and k in tokenizer_config:
+            setattr(tokenizer_cls, k, tokenizer_config[k])
+
+
+def get_model_tokenizer_chatglm(model_dir: str,
+                                model_info: ModelInfo,
+                                model_kwargs: Dict[str, Any],
+                                load_model: bool = True,
+                                **kwargs):
+    if model_kwargs.get('quantization_config') is not None:
+        model_kwargs['quantization_config'].llm_int8_skip_modules = ['output_layer']
+    # fix transformers>=4.34 bug
+    if version.parse(transformers.__version__) >= version.parse('4.34'):
+        tokenizer_config = get_tokenizer_config(model_dir)
+        class_ref = tokenizer_config['auto_map']['AutoTokenizer'][0]
+        tokenizer_cls: Type[PreTrainedTokenizerBase] = get_class_from_dynamic_module(class_ref, model_dir)
+        tokenizer_cls._auto_class = 'AutoTokenizer'
+        remove_property(tokenizer_cls, tokenizer_config)
+        kwargs['tokenizer'] = tokenizer_cls.from_pretrained(model_dir, trust_remote_code=True)
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if model is not None:
+        from torch.nn import CrossEntropyLoss
+        __old_forward = CrossEntropyLoss.forward
+
+        def cross_entropy_forward(self, inputs: torch.Tensor, target: torch.Tensor) -> torch.Tensor:
+            target = target.to(device=inputs.device)
+            return __old_forward(self, inputs, target)
+
+        CrossEntropyLoss.forward = cross_entropy_forward
+
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.chatglm2, [
+            ModelGroup([
+                Model('ZhipuAI/chatglm2-6b', 'THUDM/chatglm2-6b'),
+                Model('ZhipuAI/chatglm2-6b-32k', 'THUDM/chatglm2-6b-32k')
+            ],
+                       requires=['transformers<4.42']),
+            ModelGroup(
+                [Model('ZhipuAI/codegeex2-6b', 'THUDM/codegeex2-6b')],
+                requires=['transformers<4.34'],
+                tags=['coding'],
+            ),
+        ],
+        TemplateType.chatglm2,
+        get_model_tokenizer_chatglm,
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        model_arch=ModelArch.chatglm))
+
+register_model(
+    ModelMeta(
+        LLMModelType.chatglm3, [
+            ModelGroup([
+                Model('ZhipuAI/chatglm3-6b', 'THUDM/chatglm3-6b'),
+                Model('ZhipuAI/chatglm3-6b-base', 'THUDM/chatglm3-6b-base'),
+                Model('ZhipuAI/chatglm3-6b-32k', 'THUDM/chatglm3-6b-32k'),
+                Model('ZhipuAI/chatglm3-6b-128k', 'THUDM/chatglm3-6b-128k'),
+            ])
+        ],
+        TemplateType.glm4,
+        get_model_tokenizer_chatglm,
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        requires=['transformers<4.42'],
+        model_arch=ModelArch.chatglm))
+
+
+def get_model_tokenizer_glm4(model_dir: str,
+                             model_info: ModelInfo,
+                             model_kwargs: Dict[str, Any],
+                             load_model: bool = True,
+                             **kwargs):
+    model_config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    AttnImpl.update_attn_impl(model_config, kwargs.get('attn_impl'))
+    kwargs['model_config'] = model_config
+    model, tokenizer = get_model_tokenizer_chatglm(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if len(tokenizer.encode('<|user|>', add_special_tokens=False)) > 1:
+        for k in tokenizer.special_tokens.keys():
+            tokenizer.add_tokens(k)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.glm4,
+        [
+            ModelGroup([
+                Model('ZhipuAI/glm-4-9b-chat', 'THUDM/glm-4-9b-chat'),
+                Model('ZhipuAI/glm-4-9b', 'THUDM/glm-4-9b'),
+                Model('ZhipuAI/glm-4-9b-chat-1m', 'THUDM/glm-4-9b-chat-1m'),
+            ]),
+            ModelGroup([
+                Model('ZhipuAI/LongWriter-glm4-9b', 'THUDM/LongWriter-glm4-9b'),
+            ])
+        ],
+        TemplateType.glm4,
+        get_model_tokenizer_glm4,
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        model_arch=ModelArch.chatglm,
+        requires=['transformers>=4.42'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.glm4_0414,
+        [
+            ModelGroup([
+                Model('ZhipuAI/GLM-4-9B-0414', 'THUDM/GLM-4-9B-0414'),
+                Model('ZhipuAI/GLM-4-32B-0414', 'THUDM/GLM-4-32B-0414'),
+                Model('ZhipuAI/GLM-4-32B-Base-0414', 'THUDM/GLM-4-32B-Base-0414'),
+                Model('ZhipuAI/GLM-Z1-9B-0414', 'THUDM/GLM-Z1-9B-0414'),
+                Model('ZhipuAI/GLM-Z1-32B-0414', 'THUDM/GLM-Z1-32B-0414'),
+            ])
+        ],
+        TemplateType.glm4_0414,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Glm4ForCausalLM'],
+        model_arch=ModelArch.chatglm,
+        requires=['transformers>=4.51'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.glm4_z1_rumination,
+        [ModelGroup([
+            Model('ZhipuAI/GLM-Z1-Rumination-32B-0414', 'THUDM/GLM-Z1-Rumination-32B-0414'),
+        ])],
+        TemplateType.glm4_z1_rumination,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Glm4ForCausalLM'],
+        model_arch=ModelArch.chatglm,
+        requires=['transformers>4.51'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.longwriter_llama3_1,
+        [ModelGroup([
+            Model('ZhipuAI/LongWriter-llama3.1-8b', 'THUDM/LongWriter-llama3.1-8b'),
+        ])],
+        TemplateType.longwriter_llama,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['LlamaForCausalLM'],
+        requires=['transformers>=4.43'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.codegeex4,
+        [ModelGroup([
+            Model('ZhipuAI/codegeex4-all-9b', 'THUDM/codegeex4-all-9b'),
+        ])],
+        TemplateType.codegeex4,
+        get_model_tokenizer_glm4,
+        requires=['transformers<4.42'],
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        model_arch=ModelArch.chatglm,
+        tags=['coding'],
+    ))
+
+
+def get_model_tokenizer_glm4v(model_dir: str,
+                              model_info: ModelInfo,
+                              model_kwargs: Dict[str, Any],
+                              load_model: bool = True,
+                              **kwargs):
+    model, tokenizer = get_model_tokenizer_glm4(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    # fix merge-lora
+    tokenizer.init_kwargs['image_size'] = 1120
+    if load_model:
+        # fix device_map 4
+        n_gpu = get_device_count()
+        local_world_size = get_dist_setting()[3]
+        if n_gpu // local_world_size >= 4:
+            for layer in model.transformer.vision.transformer.layers:
+                patch_output_to_input_device(layer.mlp)
+                patch_output_to_input_device(layer.post_attention_layernorm)
+            device = next(model.transformer.vision.linear_proj.parameters()).device
+            model.transformer.vision.boi.data = model.transformer.vision.boi.to(device)
+            model.transformer.vision.eoi.data = model.transformer.vision.eoi.to(device)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.glm4v,
+        [
+            ModelGroup(
+                [
+                    Model('ZhipuAI/glm-4v-9b', 'THUDM/glm-4v-9b'),
+                ],
+                requires=['transformers>=4.42,<4.45'],
+            ),
+            ModelGroup(
+                [
+                    Model('ZhipuAI/cogagent-9b-20241220', 'THUDM/cogagent-9b-20241220'),
+                ],
+                requires=['transformers>=4.42'],
+            )
+        ],
+        TemplateType.glm4v,
+        get_model_tokenizer_glm4v,
+        architectures=['ChatGLMModel', 'ChatGLMForConditionalGeneration'],
+        model_arch=ModelArch.glm4v,
+    ))
+
+
+def get_model_tokenizer_cogvlm(model_dir: str,
+                               model_info: ModelInfo,
+                               model_kwargs: Dict[str, Any],
+                               load_model: bool = True,
+                               **kwargs):
+    tokenizer_dir = safe_snapshot_download('AI-ModelScope/vicuna-7b-v1.5', download_model=False, check_local=True)
+    tokenizer = AutoTokenizer.from_pretrained(tokenizer_dir, trust_remote_code=True)
+    if load_model:
+        logger.warning('CogAgent with FusedLayerNorm will cause an training loss of NAN, '
+                       'to avoid this, please uninstall apex.')
+        logger.info('Please ignore the unimported warning.')
+    model, tokenizer = get_model_tokenizer_with_flash_attn(
+        model_dir, model_info, model_kwargs, load_model, tokenizer=tokenizer, **kwargs)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogvlm, [
+            ModelGroup([
+                Model('ZhipuAI/cogvlm-chat', 'THUDM/cogvlm-chat-hf'),
+            ]),
+        ],
+        TemplateType.cogvlm,
+        get_model_tokenizer_cogvlm,
+        architectures=['CogVLMForCausalLM'],
+        requires=['transformers<4.42'],
+        model_arch=ModelArch.cogvlm))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogagent_chat, [
+            ModelGroup([
+                Model('ZhipuAI/cogagent-chat', 'THUDM/cogagent-chat-hf'),
+            ]),
+        ],
+        TemplateType.cogagent_chat,
+        get_model_tokenizer_cogvlm,
+        architectures=['CogAgentForCausalLM'],
+        requires=['transformers<4.42', 'timm'],
+        model_arch=ModelArch.cogvlm))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogagent_vqa, [ModelGroup([
+            Model('ZhipuAI/cogagent-vqa', 'THUDM/cogagent-vqa-hf'),
+        ])],
+        TemplateType.cogagent_vqa,
+        get_model_tokenizer_cogvlm,
+        architectures=['CogAgentForCausalLM'],
+        requires=['transformers<4.42'],
+        model_arch=ModelArch.cogvlm))
+
+
+def get_model_tokenizer_cogvlm2(*args, **kwargs):
+    model, tokenizer = get_model_tokenizer_with_flash_attn(*args, **kwargs)
+    if model is not None:
+        # fix device map 4
+        for layer in model.model.vision.transformer.layers:
+            patch_output_to_input_device(layer.mlp)
+            patch_output_to_input_device(layer.post_attention_layernorm)
+
+        device = next(model.model.vision.linear_proj.parameters()).device
+        model.model.vision.boi.data = model.model.vision.boi.to(device)
+        model.model.vision.eoi.data = model.model.vision.eoi.to(device)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogvlm2, [
+            ModelGroup([
+                Model('ZhipuAI/cogvlm2-llama3-chat-19B', 'THUDM/cogvlm2-llama3-chat-19B'),
+                Model('ZhipuAI/cogvlm2-llama3-chinese-chat-19B', 'THUDM/cogvlm2-llama3-chinese-chat-19B'),
+            ]),
+        ],
+        TemplateType.cogvlm2,
+        get_model_tokenizer_cogvlm2,
+        architectures=['CogVLMForCausalLM'],
+        requires=['transformers<4.42'],
+        model_arch=ModelArch.cogvlm))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.cogvlm2_video,
+        [
+            ModelGroup([
+                Model('ZhipuAI/cogvlm2-video-llama3-chat', 'THUDM/cogvlm2-video-llama3-chat'),
+            ]),
+        ],
+        TemplateType.cogvlm2_video,
+        get_model_tokenizer_cogvlm2,
+        architectures=['CogVLMVideoForCausalLM'],
+        requires=['decord', 'pytorchvideo', 'transformers>=4.42'],
+        model_arch=ModelArch.cogvlm,
+        tags=['video'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.glm_edge,
+        [
+            ModelGroup([
+                Model('ZhipuAI/glm-edge-1.5b-chat', 'THUDM/glm-edge-1.5b-chat'),
+                Model('ZhipuAI/glm-edge-4b-chat', 'THUDM/glm-edge-4b-chat'),
+            ]),
+        ],
+        TemplateType.glm4,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['GlmForCausalLM'],
+        requires=['transformers>=4.46'],
+    ))
+
+
+def get_model_tokenizer_glm_edge_v(model_dir: str, *args, **kwargs):
+    from transformers import AutoImageProcessor
+    processor = AutoImageProcessor.from_pretrained(model_dir)
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, *args, **kwargs)
+    processor.tokenizer = tokenizer
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.glm_edge_v,
+        [
+            ModelGroup([
+                Model('ZhipuAI/glm-edge-v-2b', 'THUDM/glm-edge-v-2b'),
+                Model('ZhipuAI/glm-edge-4b-chat', 'THUDM/glm-edge-4b-chat'),
+            ]),
+        ],
+        TemplateType.glm_edge_v,
+        get_model_tokenizer_glm_edge_v,
+        architectures=['GlmForCausalLM'],
+        requires=['transformers>=4.46'],
+        model_arch=ModelArch.glm_edge_v,
+        tags=['vision'],
+    ))

ms-swift/swift/llm/model/model/minimax.py

@@ -0,0 +1,156 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from typing import Any, Dict
+
+import json
+from transformers import AutoConfig, AutoProcessor
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from swift.llm import TemplateType
+from swift.utils import get_device, get_device_count, get_dist_setting, get_logger
+from ..constant import LLMModelType, MLLMModelType
+from ..patcher import patch_ignore_check_imports
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import ModelInfo
+
+logger = get_logger()
+
+
+def get_model_tokenizer_minimax_vl(model_dir: str,
+                                   model_info: ModelInfo,
+                                   model_kwargs: Dict[str, Any],
+                                   load_model: bool = True,
+                                   **kwargs):
+    logger.warn('NOTE: minimax-vl-01 model does not support training.')
+    n_gpu = get_device_count()
+    _, local_rank, _, local_world_size = get_dist_setting()
+    device_ids = list(range(max(local_rank, 0), n_gpu, local_world_size))
+    config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    kwargs['model_config'] = config
+    if kwargs.get('attn_impl') == 'flash_attn':
+        config.attn_type_list = [1] * len(config.attn_type_list)
+    else:
+        config.attn_type_list = [0] * len(config.attn_type_list)
+    if 'quantization_config' in model_kwargs:
+        quantization_config = model_kwargs['quantization_config']
+        from transformers import QuantoConfig
+        if isinstance(quantization_config, QuantoConfig):
+            quantization_config.modules_to_not_convert = (
+                [
+                    'vision_tower',
+                    'image_newline',
+                    'multi_modal_projector',
+                    'lm_head',
+                    'embed_tokens',
+                ] + [f'model.layers.{i}.coefficient' for i in range(config.text_config.num_hidden_layers)]
+                + [f'model.layers.{i}.block_sparse_moe.gate' for i in range(config.text_config.num_hidden_layers)])
+
+    if len(device_ids) > 1:
+        model_safetensors_index_path = os.path.join(model_dir, 'model.safetensors.index.json')
+        with open(model_safetensors_index_path, 'r') as f:
+            model_safetensors_index = json.load(f)
+        weight_map = model_safetensors_index['weight_map']
+        vision_map = {}
+        for key, value in weight_map.items():
+            if 'vision_tower' in key or 'image_newline' in key or 'multi_modal_projector' in key:
+                new_key = key.replace('.weight', '').replace('.bias', '')
+                if new_key not in vision_map:
+                    vision_map[new_key] = value
+
+        device_map = {
+            'language_model.model.embed_tokens': get_device(device_ids[0]),
+            'language_model.model.norm': get_device(device_ids[len(device_ids) - 1]),
+            'language_model.lm_head': get_device(device_ids[len(device_ids) - 1])
+        }
+        for key, value in vision_map.items():
+            device_map[key] = get_device(device_ids[0])
+        device_map['vision_tower.vision_model.post_layernorm'] = get_device(device_ids[0])
+        layers_per_device = config.text_config.num_hidden_layers // len(device_ids)
+        for i in range(len(device_ids)):
+            for j in range(layers_per_device):
+                device_map[f'language_model.model.layers.{i * layers_per_device + j}'] = get_device(device_ids[i])
+        model_kwargs['device_map'] = device_map
+
+    MiniMaxVL01ProcessorKwargs = get_class_from_dynamic_module('processing_minimax_vl_01.MiniMaxVL01ProcessorKwargs',
+                                                               model_dir)
+    get_hw_multiple_of = get_class_from_dynamic_module('processing_minimax_vl_01.get_hw_multiple_of', model_dir)
+    get_num_token = get_class_from_dynamic_module('processing_minimax_vl_01.get_num_token', model_dir)
+
+    processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+    processor.MiniMaxVL01ProcessorKwargs = MiniMaxVL01ProcessorKwargs
+    processor.get_hw_multiple_of = get_hw_multiple_of
+    processor.get_num_token = get_num_token
+    with patch_ignore_check_imports():
+        model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model,
+                                                               **kwargs)
+    processor.tokenizer = tokenizer
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minimax_vl, [
+            ModelGroup([
+                Model('MiniMax/MiniMax-VL-01', 'MiniMaxAI/MiniMax-VL-01'),
+            ]),
+        ],
+        TemplateType.minimax_vl,
+        get_model_tokenizer_minimax_vl,
+        architectures=['MiniMaxVL01ForConditionalGeneration'],
+        tags=['vision']))
+
+
+def get_model_tokenizer_minimax_text(model_dir: str,
+                                     model_info: ModelInfo,
+                                     model_kwargs: Dict[str, Any],
+                                     load_model: bool = True,
+                                     **kwargs):
+    logger.warn('NOTE: minimax-text-01 model does not support training.')
+    n_gpu = get_device_count()
+    _, local_rank, _, local_world_size = get_dist_setting()
+    device_ids = list(range(max(local_rank, 0), n_gpu, local_world_size))
+    config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    kwargs['model_config'] = config
+    if kwargs.get('attn_impl') == 'flash_attn':
+        config.attn_type_list = [1] * len(config.attn_type_list)
+    else:
+        config.attn_type_list = [0] * len(config.attn_type_list)
+    if 'quantization_config' in model_kwargs:
+        quantization_config = model_kwargs['quantization_config']
+        from transformers import QuantoConfig
+        if isinstance(quantization_config, QuantoConfig):
+            quantization_config.modules_to_not_convert = (
+                [
+                    'lm_head',
+                    'embed_tokens',
+                ] + [f'model.layers.{i}.coefficient' for i in range(config.num_hidden_layers)]
+                + [f'model.layers.{i}.block_sparse_moe.gate' for i in range(config.num_hidden_layers)])
+
+    if len(device_ids) > 1:
+        layers_per_device = config.num_hidden_layers // len(device_ids)
+        # set device map
+        device_map = {
+            'model.embed_tokens': get_device(0),
+            'model.norm': get_device(len(device_ids) - 1),
+            'lm_head': get_device(len(device_ids) - 1)
+        }
+        for i in range(len(device_ids)):
+            for j in range(layers_per_device):
+                device_map[f'model.layers.{i * layers_per_device + j}'] = get_device(i)
+        model_kwargs['device_map'] = device_map
+    with patch_ignore_check_imports():
+        model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model,
+                                                               **kwargs)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.minimax, [
+            ModelGroup([
+                Model('MiniMax/MiniMax-Text-01', 'MiniMaxAI/MiniMax-Text-01'),
+            ]),
+        ],
+        TemplateType.minimax,
+        get_model_tokenizer_minimax_text,
+        architectures=['MiniMaxText01ForCausalLM']))

ms-swift/swift/llm/model/model/openbuddy.py

@@ -0,0 +1,85 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from swift.llm import TemplateType
+from swift.utils import get_logger
+from ..constant import LLMModelType
+from ..model_arch import ModelArch
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+
+logger = get_logger()
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_llama,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama-65b-v8-bf16', 'OpenBuddy/openbuddy-llama-65b-v8-bf16'),
+            ]),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama2-13b-v8.1-fp16', 'OpenBuddy/openbuddy-llama2-13b-v8.1-fp16'),
+                Model('OpenBuddy/openbuddy-llama2-70b-v10.1-bf16', 'OpenBuddy/openbuddy-llama2-70b-v10.1-bf16'),
+            ]),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-deepseek-67b-v15.2', 'OpenBuddy/openbuddy-deepseek-67b-v15.2'),
+            ]),
+        ],
+        TemplateType.openbuddy,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['LlamaForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_llama3,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama3-8b-v21.1-8k', 'OpenBuddy/openbuddy-llama3-8b-v21.1-8k'),
+                Model('OpenBuddy/openbuddy-llama3-70b-v21.1-8k', 'OpenBuddy/openbuddy-llama3-70b-v21.1-8k'),
+                Model('OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k', 'OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k'),
+            ]),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k', 'OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k'),
+                Model('OpenBuddy/openbuddy-nemotron-70b-v23.2-131k', 'OpenBuddy/openbuddy-nemotron-70b-v23.2-131k'),
+            ],
+                       requires=['transformers>=4.43']),
+            ModelGroup(
+                [Model('OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k', 'OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k')],
+                requires=['transformers>=4.45'])
+        ],
+        TemplateType.openbuddy2,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['LlamaForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_mistral,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-mistral-7b-v17.1-32k', 'OpenBuddy/openbuddy-mistral-7b-v17.1-32k'),
+            ]),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-zephyr-7b-v14.1', 'OpenBuddy/openbuddy-zephyr-7b-v14.1'),
+            ]),
+        ],
+        TemplateType.openbuddy,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.34'],
+        architectures=['MistralForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_mixtral,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k', 'OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k'),
+            ], ),
+        ],
+        TemplateType.openbuddy,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MixtralForCausalLM'],
+        requires=['transformers>=4.36'],
+    ))

ms-swift/swift/llm/model/model/qwen.py

@@ -0,0 +1,899 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, Optional, Tuple, Type
+
+import torch
+from transformers import AutoConfig, BitsAndBytesConfig, PreTrainedTokenizerBase
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from transformers.models.auto.tokenization_auto import get_tokenizer_config
+
+from swift.llm import TemplateType
+from swift.utils import get_device_count, get_dist_setting, get_env_args, get_logger
+from ..constant import LLMModelType, MLLMModelType, RMModelType
+from ..model_arch import ModelArch
+from ..patcher import patch_fixed_device, patch_output_clone, patch_output_to_input_device
+from ..register import (Model, ModelGroup, ModelMeta, get_model_tokenizer_multimodal, get_model_tokenizer_reward_model,
+                        get_model_tokenizer_with_flash_attn, register_model)
+from ..utils import AttnImpl, ModelInfo, use_submodel_func
+
+logger = get_logger()
+dtype_mapping = {torch.float16: 'fp16', torch.bfloat16: 'bf16', torch.float32: 'fp32'}
+
+
+def get_model_tokenizer_qwen(model_dir: str,
+                             model_info: ModelInfo,
+                             model_kwargs: Dict[str, Any],
+                             load_model: bool = True,
+                             model_config=None,
+                             **kwargs):
+    if model_config is None:
+        model_config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    if model_info.torch_dtype is not None:
+        k_true = dtype_mapping[model_info.torch_dtype]
+        for k in dtype_mapping.values():
+            setattr(model_config, k, k == k_true)
+
+    quantization_config = model_kwargs.get('quantization_config')
+    if not isinstance(quantization_config, BitsAndBytesConfig):
+        # not bnb quant
+        model_config.torch_dtype = None
+    use_flash_attn = AttnImpl.to_use_flash_attn(kwargs.pop('attn_impl', None), 'auto')
+    model_config.use_flash_attn = use_flash_attn
+    kwargs['model_config'] = model_config
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    try:
+        # fix mp+ddp bug
+        model.transformer.registered_causal_mask = model.transformer.registered_causal_mask.cuda()
+        logger.info('registered_causal_mask to cuda')
+    except AttributeError:
+        pass
+    if tokenizer.eos_token_id is None:
+        tokenizer.eos_token_id = tokenizer.eod_id
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen,
+        [
+            # qwen
+            ModelGroup([
+                # chat
+                Model('Qwen/Qwen-1_8B-Chat', 'Qwen/Qwen-1_8B-Chat'),
+                Model('Qwen/Qwen-7B-Chat', 'Qwen/Qwen-7B-Chat'),
+                Model('Qwen/Qwen-14B-Chat', 'Qwen/Qwen-14B-Chat'),
+                Model('Qwen/Qwen-72B-Chat', 'Qwen/Qwen-72B-Chat'),
+                # base
+                Model('Qwen/Qwen-1_8B', 'Qwen/Qwen-1_8B'),
+                Model('Qwen/Qwen-7B', 'Qwen/Qwen-7B'),
+                Model('Qwen/Qwen-14B', 'Qwen/Qwen-14B'),
+                Model('Qwen/Qwen-72B', 'Qwen/Qwen-72B'),
+                # gptq-int4
+                Model('Qwen/Qwen-1_8B-Chat-Int4', 'Qwen/Qwen-1_8B-Chat-Int4'),
+                Model('Qwen/Qwen-7B-Chat-Int4', 'Qwen/Qwen-7B-Chat-Int4'),
+                Model('Qwen/Qwen-14B-Chat-Int4', 'Qwen/Qwen-14B-Chat-Int4'),
+                Model('Qwen/Qwen-72B-Chat-Int4', 'Qwen/Qwen-72B-Chat-Int4'),
+                # gptq-int8
+                Model('Qwen/Qwen-1_8B-Chat-Int8', 'Qwen/Qwen-1_8B-Chat-Int8'),
+                Model('Qwen/Qwen-7B-Chat-Int8', 'Qwen/Qwen-7B-Chat-Int8'),
+                Model('Qwen/Qwen-14B-Chat-Int8', 'Qwen/Qwen-14B-Chat-Int8'),
+                Model('Qwen/Qwen-72B-Chat-Int8', 'Qwen/Qwen-72B-Chat-Int8'),
+            ]),
+            # tongyi-finance
+            ModelGroup([
+                Model('TongyiFinance/Tongyi-Finance-14B-Chat', 'jxy/Tongyi-Finance-14B-Chat'),
+                Model('TongyiFinance/Tongyi-Finance-14B'),
+                Model('TongyiFinance/Tongyi-Finance-14B-Chat-Int4', 'jxy/Tongyi-Finance-14B-Chat-Int4'),
+            ],
+                       tags=['financial']),
+        ],
+        TemplateType.qwen,
+        get_model_tokenizer_qwen,
+        architectures=['QWenLMHeadModel'],
+        model_arch=ModelArch.qwen))
+
+register_model(
+    ModelMeta(
+        LLMModelType.modelscope_agent,
+        [ModelGroup([
+            Model('iic/ModelScope-Agent-7B'),
+            Model('iic/ModelScope-Agent-14B'),
+        ])],
+        TemplateType.modelscope_agent,
+        get_model_tokenizer_qwen,
+        architectures=['QWenLMHeadModel'],
+        model_arch=ModelArch.qwen))
+
+
+def _qwen_vl_audio_decode(self, *args, skip_special_tokens=False, **kwargs) -> str:
+    if skip_special_tokens:
+        token_ids = kwargs['token_ids']
+        while len(token_ids) > 0 and token_ids[-1] in {151645, 151643}:
+            token_ids.pop()
+        return self._old_decode(*args, skip_special_tokens=False, **kwargs)
+    else:
+        return self._old_decode(*args, skip_special_tokens=False, **kwargs)
+
+
+def fix_qwen_inplace_bug(model) -> None:
+    # qwen-vl, qwen-audio
+    first_drop = model.transformer.drop
+    if first_drop.p == 0.:
+        # fix in-place operation bug
+        patch_output_clone(first_drop)
+
+
+def get_model_tokenizer_qwen_audio(model_dir: str,
+                                   model_info: ModelInfo,
+                                   model_kwargs: Dict[str, Any],
+                                   load_model: bool = True,
+                                   **kwargs):
+    tokenizer_config = get_tokenizer_config(model_dir)
+    class_ref = tokenizer_config['auto_map']['AutoTokenizer'][0]
+    tokenizer_cls: Type[PreTrainedTokenizerBase] = get_class_from_dynamic_module(class_ref, model_dir)
+    tokenizer_cls._auto_class = 'AutoTokenizer'
+    tokenizer_cls.AUDIO_ST = ()  # fix no attr `self.AUDIO_ST` bug
+    if not hasattr(tokenizer_cls, '_old_decode'):
+        tokenizer_cls._old_decode = tokenizer_cls._decode
+        tokenizer_cls._decode = _qwen_vl_audio_decode
+    kwargs['tokenizer'] = tokenizer_cls.from_pretrained(model_dir, trust_remote_code=True)
+    model, tokenizer = get_model_tokenizer_qwen(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if model is not None:
+        fix_qwen_inplace_bug(model)
+
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen_audio, [
+            ModelGroup([
+                Model('Qwen/Qwen-Audio-Chat', 'Qwen/Qwen-Audio-Chat'),
+                Model('Qwen/Qwen-Audio', 'Qwen/Qwen-Audio'),
+            ])
+        ],
+        TemplateType.qwen_audio,
+        get_model_tokenizer_qwen_audio,
+        model_arch=ModelArch.qwen_audio,
+        architectures=['QWenLMHeadModel'],
+        additional_saved_files=['mel_filters.npz'],
+        tags=['audio']))
+
+
+def _qwen_vl_visual_block_forward(
+    self,
+    q_x: torch.Tensor,
+    k_x: Optional[torch.Tensor] = None,
+    v_x: Optional[torch.Tensor] = None,
+    attn_mask: Optional[torch.Tensor] = None,
+):
+    k_x = self.ln_1_kv(k_x) if hasattr(self, 'ln_1_kv') and k_x is not None else None
+    v_x = self.ln_1_kv(v_x) if hasattr(self, 'ln_1_kv') and v_x is not None else None
+
+    x = q_x + self.attention(q_x=self.ln_1(q_x), k_x=k_x, v_x=v_x, attn_mask=attn_mask)
+    z = self.mlp(self.ln_2(x))
+    x = x.to(z.device) + z  # FIX
+    return x
+
+
+def get_model_tokenizer_qwen_vl(model_dir: str,
+                                model_info: ModelInfo,
+                                model_kwargs: Dict[str, Any],
+                                load_model: bool = True,
+                                **kwargs):
+    if (model_kwargs.get('quantization_config') is not None
+            and isinstance(model_kwargs['quantization_config'], BitsAndBytesConfig)):
+        # https://github.com/pytorch/pytorch/issues/58969
+        model_kwargs['quantization_config'].llm_int8_skip_modules = ['lm_head', 'attn_pool.attn']
+        _TransformerBlock = get_class_from_dynamic_module('visual.TransformerBlock', model_dir)
+
+        def _get_cast_dtype(self) -> torch.dtype:
+            return self.resblocks[0].ln_1.weight.dtype
+
+        _TransformerBlock.__old_get_cast_dtype = _TransformerBlock.get_cast_dtype
+        _TransformerBlock.get_cast_dtype = _get_cast_dtype
+
+    tokenizer_config = get_tokenizer_config(model_dir)
+    class_ref = tokenizer_config['auto_map']['AutoTokenizer'][0]
+    tokenizer_cls: Type[PreTrainedTokenizerBase] = get_class_from_dynamic_module(class_ref, model_dir)
+    tokenizer_cls._auto_class = 'AutoTokenizer'
+    tokenizer_cls.IMAGE_ST = ()  # fix no attr `self.IMAGE_ST` bug
+    if not hasattr(tokenizer_cls, '_old_decode'):
+        tokenizer_cls._old_decode = tokenizer_cls._decode
+        tokenizer_cls._decode = _qwen_vl_audio_decode
+    # fix device_map is 4
+    n_gpu = get_device_count()
+    local_world_size = get_dist_setting()[3]
+    if n_gpu // local_world_size >= 4:
+        visual_block_cls = get_class_from_dynamic_module('visual.VisualAttentionBlock', model_dir)
+        visual_block_cls.__old_forward = visual_block_cls.forward
+        visual_block_cls.forward = _qwen_vl_visual_block_forward
+
+    kwargs['tokenizer'] = tokenizer_cls.from_pretrained(model_dir, trust_remote_code=True)
+    model, tokenizer = get_model_tokenizer_qwen(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if model is not None:
+        device_type = next(model.parameters()).device.type
+        fix_qwen_inplace_bug(model)
+        # fix device_map is 4
+        if n_gpu // local_world_size >= 4:
+            model.transformer.visual.proj.data = model.transformer.visual.proj.to(
+                model.transformer.visual.ln_post.bias.device)
+        # fix images cuda:1 bug
+        patch_fixed_device(model.transformer.visual, f'{device_type}:0')
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen_vl, [
+            ModelGroup([
+                Model('Qwen/Qwen-VL-Chat', 'Qwen/Qwen-VL-Chat'),
+                Model('Qwen/Qwen-VL', 'Qwen/Qwen-VL'),
+                Model('Qwen/Qwen-VL-Chat-Int4', 'Qwen/Qwen-VL-Chat-Int4'),
+            ])
+        ],
+        TemplateType.qwen_vl,
+        get_model_tokenizer_qwen_vl,
+        model_arch=ModelArch.qwen_vl,
+        architectures=['QWenLMHeadModel'],
+        additional_saved_files=['SimSun.ttf'],
+        tags=['vision']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen2,
+        [
+            # qwen1.5
+            ModelGroup([
+                # chat
+                Model('Qwen/Qwen1.5-0.5B-Chat', 'Qwen/Qwen1.5-0.5B-Chat'),
+                Model('Qwen/Qwen1.5-1.8B-Chat', 'Qwen/Qwen1.5-1.8B-Chat'),
+                Model('Qwen/Qwen1.5-4B-Chat', 'Qwen/Qwen1.5-4B-Chat'),
+                Model('Qwen/Qwen1.5-7B-Chat', 'Qwen/Qwen1.5-7B-Chat'),
+                Model('Qwen/Qwen1.5-14B-Chat', 'Qwen/Qwen1.5-14B-Chat'),
+                Model('Qwen/Qwen1.5-32B-Chat', 'Qwen/Qwen1.5-32B-Chat'),
+                Model('Qwen/Qwen1.5-72B-Chat', 'Qwen/Qwen1.5-72B-Chat'),
+                Model('Qwen/Qwen1.5-110B-Chat', 'Qwen/Qwen1.5-110B-Chat'),
+                # base
+                Model('Qwen/Qwen1.5-0.5B', 'Qwen/Qwen1.5-0.5B'),
+                Model('Qwen/Qwen1.5-1.8B', 'Qwen/Qwen1.5-1.8B'),
+                Model('Qwen/Qwen1.5-4B', 'Qwen/Qwen1.5-4B'),
+                Model('Qwen/Qwen1.5-7B', 'Qwen/Qwen1.5-7B'),
+                Model('Qwen/Qwen1.5-14B', 'Qwen/Qwen1.5-14B'),
+                Model('Qwen/Qwen1.5-32B', 'Qwen/Qwen1.5-32B'),
+                Model('Qwen/Qwen1.5-72B', 'Qwen/Qwen1.5-72B'),
+                Model('Qwen/Qwen1.5-110B', 'Qwen/Qwen1.5-110B'),
+                # gptq-int4
+                Model('Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-4B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-7B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-14B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-32B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-72B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-110B-Chat-GPTQ-Int4'),
+                # gptq-int8
+                Model('Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-4B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-7B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-14B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-72B-Chat-GPTQ-Int8'),
+                # awq-int4
+                Model('Qwen/Qwen1.5-0.5B-Chat-AWQ', 'Qwen/Qwen1.5-0.5B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-1.8B-Chat-AWQ', 'Qwen/Qwen1.5-1.8B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-4B-Chat-AWQ', 'Qwen/Qwen1.5-4B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-7B-Chat-AWQ', 'Qwen/Qwen1.5-7B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-14B-Chat-AWQ', 'Qwen/Qwen1.5-14B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-32B-Chat-AWQ', 'Qwen/Qwen1.5-32B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-72B-Chat-AWQ', 'Qwen/Qwen1.5-72B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-110B-Chat-AWQ', 'Qwen/Qwen1.5-110B-Chat-AWQ'),
+            ]),
+            # code-qwen1.5
+            ModelGroup([
+                Model('Qwen/CodeQwen1.5-7B', 'Qwen/CodeQwen1.5-7B'),
+                Model('Qwen/CodeQwen1.5-7B-Chat', 'Qwen/CodeQwen1.5-7B-Chat'),
+                Model('Qwen/CodeQwen1.5-7B-Chat-AWQ', 'Qwen/CodeQwen1.5-7B-Chat-AWQ'),
+            ],
+                       tags=['coding']),
+            # qwen2
+            ModelGroup([
+                # instruct
+                Model('Qwen/Qwen2-0.5B-Instruct', 'Qwen/Qwen2-0.5B-Instruct'),
+                Model('Qwen/Qwen2-1.5B-Instruct', 'Qwen/Qwen2-1.5B-Instruct'),
+                Model('Qwen/Qwen2-7B-Instruct', 'Qwen/Qwen2-7B-Instruct'),
+                Model('Qwen/Qwen2-72B-Instruct', 'Qwen/Qwen2-72B-Instruct'),
+                # base
+                Model('Qwen/Qwen2-0.5B', 'Qwen/Qwen2-0.5B'),
+                Model('Qwen/Qwen2-1.5B', 'Qwen/Qwen2-1.5B'),
+                Model('Qwen/Qwen2-7B', 'Qwen/Qwen2-7B'),
+                Model('Qwen/Qwen2-72B', 'Qwen/Qwen2-72B'),
+                # gptq-int4
+                Model('Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2-7B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-7B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2-72B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-72B-Instruct-GPTQ-Int4'),
+                # gptq-int8
+                Model('Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2-7B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-7B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2-72B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-72B-Instruct-GPTQ-Int8'),
+                # awq-int4
+                Model('Qwen/Qwen2-0.5B-Instruct-AWQ', 'Qwen/Qwen2-0.5B-Instruct-AWQ'),
+                Model('Qwen/Qwen2-1.5B-Instruct-AWQ', 'Qwen/Qwen2-1.5B-Instruct-AWQ'),
+                Model('Qwen/Qwen2-7B-Instruct-AWQ', 'Qwen/Qwen2-7B-Instruct-AWQ'),
+                Model('Qwen/Qwen2-72B-Instruct-AWQ', 'Qwen/Qwen2-72B-Instruct-AWQ'),
+            ]),
+            # qwen2-math
+            ModelGroup(
+                [
+                    # instruct
+                    Model('Qwen/Qwen2-Math-1.5B-Instruct', 'Qwen/Qwen2-Math-1.5B-Instruct'),
+                    Model('Qwen/Qwen2-Math-7B-Instruct', 'Qwen/Qwen2-Math-7B-Instruct'),
+                    Model('Qwen/Qwen2-Math-72B-Instruct', 'Qwen/Qwen2-Math-72B-Instruct'),
+                    # base
+                    Model('Qwen/Qwen2-Math-1.5B', 'Qwen/Qwen2-Math-1.5B'),
+                    Model('Qwen/Qwen2-Math-7B', 'Qwen/Qwen2-Math-7B'),
+                    Model('Qwen/Qwen2-Math-72B', 'Qwen/Qwen2-Math-72B'),
+                ],
+                tags=['math']),
+            # qwen2.5-1m
+            ModelGroup([
+                Model('Qwen/Qwen2.5-7B-Instruct-1M', 'Qwen/Qwen2.5-7B-Instruct-1M'),
+                Model('Qwen/Qwen2.5-14B-Instruct-1M', 'Qwen/Qwen2.5-14B-Instruct-1M'),
+            ]),
+            # other
+            ModelGroup([Model('PowerInfer/SmallThinker-3B-Preview', 'PowerInfer/SmallThinker-3B-Preview')]),
+        ],
+        TemplateType.qwen,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen2_5,
+        [
+            # qwen2.5
+            ModelGroup([
+                # instruct
+                Model('Qwen/Qwen2.5-0.5B-Instruct', 'Qwen/Qwen2.5-0.5B-Instruct'),
+                Model('Qwen/Qwen2.5-1.5B-Instruct', 'Qwen/Qwen2.5-1.5B-Instruct'),
+                Model('Qwen/Qwen2.5-3B-Instruct', 'Qwen/Qwen2.5-3B-Instruct'),
+                Model('Qwen/Qwen2.5-7B-Instruct', 'Qwen/Qwen2.5-7B-Instruct'),
+                Model('Qwen/Qwen2.5-14B-Instruct', 'Qwen/Qwen2.5-14B-Instruct'),
+                Model('Qwen/Qwen2.5-32B-Instruct', 'Qwen/Qwen2.5-32B-Instruct'),
+                Model('Qwen/Qwen2.5-72B-Instruct', 'Qwen/Qwen2.5-72B-Instruct'),
+                # base
+                Model('Qwen/Qwen2.5-0.5B', 'Qwen/Qwen2.5-0.5B'),
+                Model('Qwen/Qwen2.5-1.5B', 'Qwen/Qwen2.5-1.5B'),
+                Model('Qwen/Qwen2.5-3B', 'Qwen/Qwen2.5-3B'),
+                Model('Qwen/Qwen2.5-7B', 'Qwen/Qwen2.5-7B'),
+                Model('Qwen/Qwen2.5-14B', 'Qwen/Qwen2.5-14B'),
+                Model('Qwen/Qwen2.5-32B', 'Qwen/Qwen2.5-32B'),
+                Model('Qwen/Qwen2.5-72B', 'Qwen/Qwen2.5-72B'),
+                # gptq-int4
+                Model('Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4'),
+                # gptq-int8
+                Model('Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8'),
+                # awq-int4
+                Model('Qwen/Qwen2.5-0.5B-Instruct-AWQ', 'Qwen/Qwen2.5-0.5B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-1.5B-Instruct-AWQ', 'Qwen/Qwen2.5-1.5B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-3B-Instruct-AWQ', 'Qwen/Qwen2.5-3B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-7B-Instruct-AWQ', 'Qwen/Qwen2.5-7B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-14B-Instruct-AWQ', 'Qwen/Qwen2.5-14B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-32B-Instruct-AWQ', 'Qwen/Qwen2.5-32B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-72B-Instruct-AWQ', 'Qwen/Qwen2.5-72B-Instruct-AWQ'),
+            ]),
+            # qwen2.5-coder
+            ModelGroup(
+                [
+                    # instruct
+                    Model('Qwen/Qwen2.5-Coder-0.5B-Instruct', 'Qwen/Qwen2.5-Coder-0.5B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B-Instruct', 'Qwen/Qwen2.5-Coder-1.5B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-3B-Instruct', 'Qwen/Qwen2.5-Coder-3B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-7B-Instruct', 'Qwen/Qwen2.5-Coder-7B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-14B-Instruct', 'Qwen/Qwen2.5-Coder-14B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-32B-Instruct', 'Qwen/Qwen2.5-Coder-32B-Instruct'),
+                    # base
+                    Model('Qwen/Qwen2.5-Coder-0.5B', 'Qwen/Qwen2.5-Coder-0.5B'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B', 'Qwen/Qwen2.5-Coder-1.5B'),
+                    Model('Qwen/Qwen2.5-Coder-3B', 'Qwen/Qwen2.5-Coder-3B'),
+                    Model('Qwen/Qwen2.5-Coder-7B', 'Qwen/Qwen2.5-Coder-7B'),
+                    Model('Qwen/Qwen2.5-Coder-14B', 'Qwen/Qwen2.5-Coder-14B'),
+                    Model('Qwen/Qwen2.5-Coder-32B', 'Qwen/Qwen2.5-Coder-32B'),
+                    # AWQ
+                    Model('Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-3B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-7B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-14B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-32B-Instruct-AWQ'),
+                    # GPTQ
+                    Model('Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8'),
+                ],
+                tags=['coding'])
+        ],
+        TemplateType.qwen2_5,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen2_5_math,
+        [
+            # qwen2.5-math
+            ModelGroup(
+                [
+                    # instruct
+                    Model('Qwen/Qwen2.5-Math-1.5B-Instruct', 'Qwen/Qwen2.5-Math-1.5B-Instruct'),
+                    Model('Qwen/Qwen2.5-Math-7B-Instruct', 'Qwen/Qwen2.5-Math-7B-Instruct'),
+                    Model('Qwen/Qwen2.5-Math-72B-Instruct', 'Qwen/Qwen2.5-Math-72B-Instruct'),
+                    # base
+                    Model('Qwen/Qwen2.5-Math-1.5B', 'Qwen/Qwen2.5-Math-1.5B'),
+                    Model('Qwen/Qwen2.5-Math-7B', 'Qwen/Qwen2.5-Math-7B'),
+                    Model('Qwen/Qwen2.5-Math-72B', 'Qwen/Qwen2.5-Math-72B'),
+                ],
+                tags=['math']),
+        ],
+        TemplateType.qwen2_5_math,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen2_moe,
+        [
+            # qwen1.5-moe
+            ModelGroup([
+                Model('Qwen/Qwen1.5-MoE-A2.7B-Chat', 'Qwen/Qwen1.5-MoE-A2.7B-Chat'),
+                Model('Qwen/Qwen1.5-MoE-A2.7B', 'Qwen/Qwen1.5-MoE-A2.7B'),
+                Model('Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4'),
+            ]),
+            ModelGroup([
+                Model('Qwen/Qwen2-57B-A14B-Instruct', 'Qwen/Qwen2-57B-A14B-Instruct'),
+                Model('Qwen/Qwen2-57B-A14B', 'Qwen/Qwen2-57B-A14B'),
+                Model('Qwen/Qwen2-57B-A14B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-57B-A14B-Instruct-GPTQ-Int4'),
+            ])
+        ],
+        TemplateType.qwen,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen2MoeForCausalLM'],
+        requires=['transformers>=4.40'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen3,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen3-0.6B-Base', 'Qwen/Qwen3-0.6B-Base'),
+                Model('Qwen/Qwen3-1.7B-Base', 'Qwen/Qwen3-1.7B-Base'),
+                Model('Qwen/Qwen3-4B-Base', 'Qwen/Qwen3-4B-Base'),
+                Model('Qwen/Qwen3-8B-Base', 'Qwen/Qwen3-8B-Base'),
+                Model('Qwen/Qwen3-14B-Base', 'Qwen/Qwen3-14B-Base'),
+                Model('Qwen/Qwen3-32B-Base', 'Qwen/Qwen3-32B-Base'),
+                # instruct
+                Model('Qwen/Qwen3-0.6B', 'Qwen/Qwen3-0.6B'),
+                Model('Qwen/Qwen3-1.7B', 'Qwen/Qwen3-1.7B'),
+                Model('Qwen/Qwen3-4B', 'Qwen/Qwen3-4B'),
+                Model('Qwen/Qwen3-8B', 'Qwen/Qwen3-8B'),
+                Model('Qwen/Qwen3-14B', 'Qwen/Qwen3-14B'),
+                Model('Qwen/Qwen3-32B', 'Qwen/Qwen3-32B'),
+                # fp8
+                Model('Qwen/Qwen3-0.6B-FP8', 'Qwen/Qwen3-0.6B-FP8'),
+                Model('Qwen/Qwen3-1.7B-FP8', 'Qwen/Qwen3-1.7B-FP8'),
+                Model('Qwen/Qwen3-4B-FP8', 'Qwen/Qwen3-4B-FP8'),
+                Model('Qwen/Qwen3-8B-FP8', 'Qwen/Qwen3-8B-FP8'),
+                Model('Qwen/Qwen3-14B-FP8', 'Qwen/Qwen3-14B-FP8'),
+                Model('Qwen/Qwen3-32B-FP8', 'Qwen/Qwen3-32B-FP8'),
+                # awq
+                Model('Qwen/Qwen3-4B-AWQ', 'Qwen/Qwen3-4B-AWQ'),
+                Model('Qwen/Qwen3-8B-AWQ', 'Qwen/Qwen3-8B-AWQ'),
+                Model('Qwen/Qwen3-14B-AWQ', 'Qwen/Qwen3-14B-AWQ'),
+                Model('Qwen/Qwen3-32B-AWQ', 'Qwen/Qwen3-32B-AWQ'),
+                # swift
+                Model('swift/Qwen3-32B-AWQ'),
+            ]),
+        ],
+        TemplateType.qwen3,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen3ForCausalLM'],
+        requires=['transformers>=4.51'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen3_moe,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen3-30B-A3B-Base', 'Qwen/Qwen3-30B-A3B-Base'),
+                Model('Qwen/Qwen3-235B-A22B-Base', 'Qwen/Qwen3-235B-A22B-Base'),
+                # instruct
+                Model('Qwen/Qwen3-30B-A3B', 'Qwen/Qwen3-30B-A3B'),
+                Model('Qwen/Qwen3-235B-A22B', 'Qwen/Qwen3-235B-A22B'),
+                # fp8
+                Model('Qwen/Qwen3-30B-A3B-FP8', 'Qwen/Qwen3-30B-A3B-FP8'),
+                Model('Qwen/Qwen3-235B-A22B-FP8', 'Qwen/Qwen3-235B-A22B-FP8'),
+                # awq
+                Model('swift/Qwen3-30B-A3B-AWQ', 'cognitivecomputations/Qwen3-30B-A3B-AWQ'),
+                Model('swift/Qwen3-235B-A22B-AWQ', 'cognitivecomputations/Qwen3-235B-A22B-AWQ'),
+            ]),
+        ],
+        TemplateType.qwen3,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen3MoeForCausalLM'],
+        requires=['transformers>=4.51'],
+    ))
+
+
+def patch_qwen_vl_utils(vision_process):
+    if hasattr(vision_process, '_patch'):
+        return
+    for key in [
+            'image_factor', 'min_pixels', 'max_pixels', 'max_ratio', 'video_min_pixels', 'video_max_pixels',
+            'video_total_pixels', 'frame_factor', 'fps', 'fps_min_frames', 'fps_max_frames'
+    ]:
+        type_func = float if key == 'fps' else int
+        setattr(vision_process, key.upper(), get_env_args(key, type_func, getattr(vision_process, key.upper())))
+    _read_video_decord = vision_process._read_video_decord
+
+    def _new_read_video_decord(ele: dict):
+        from swift.llm import load_file
+        ele['video'] = load_file(ele['video'])
+        return _read_video_decord(ele)
+
+    vision_process.VIDEO_READER_BACKENDS['decord'] = _new_read_video_decord
+    vision_process._patch = True
+
+
+def get_model_tokenizer_qwen2_vl(*args, **kwargs):
+    from transformers import Qwen2VLForConditionalGeneration
+    kwargs['automodel_class'] = kwargs['automodel_class'] or Qwen2VLForConditionalGeneration
+    model, tokenizer = get_model_tokenizer_multimodal(*args, **kwargs)
+    if model is not None and hasattr(model.model, 'embed_tokens'):
+        patch_output_clone(model.model.embed_tokens)
+        patch_output_to_input_device(model.model.embed_tokens)
+
+    from qwen_vl_utils import vision_process
+    patch_qwen_vl_utils(vision_process)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_vl,
+        [
+            ModelGroup(
+                [
+                    # chat
+                    Model('Qwen/Qwen2-VL-2B-Instruct', 'Qwen/Qwen2-VL-2B-Instruct'),
+                    Model('Qwen/Qwen2-VL-7B-Instruct', 'Qwen/Qwen2-VL-7B-Instruct'),
+                    Model('Qwen/Qwen2-VL-72B-Instruct', 'Qwen/Qwen2-VL-72B-Instruct'),
+                    # base
+                    Model('Qwen/Qwen2-VL-2B', 'Qwen/Qwen2-VL-2B'),
+                    Model('Qwen/Qwen2-VL-7B', 'Qwen/Qwen2-VL-7B'),
+                    Model('Qwen/Qwen2-VL-72B', 'Qwen/Qwen2-VL-72B'),
+                    # gptq-int4
+                    Model('Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4'),
+                    # gptq-int8
+                    Model('Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int8'),
+                    # awq-int4
+                    Model('Qwen/Qwen2-VL-2B-Instruct-AWQ', 'Qwen/Qwen2-VL-2B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2-VL-7B-Instruct-AWQ', 'Qwen/Qwen2-VL-7B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2-VL-72B-Instruct-AWQ', 'Qwen/Qwen2-VL-72B-Instruct-AWQ'),
+                ], ),
+            ModelGroup([
+                Model('bytedance-research/UI-TARS-2B-SFT', 'bytedance-research/UI-TARS-2B-SFT'),
+                Model('bytedance-research/UI-TARS-7B-SFT', 'bytedance-research/UI-TARS-7B-SFT'),
+                Model('bytedance-research/UI-TARS-7B-DPO', 'bytedance-research/UI-TARS-7B-DPO'),
+                Model('bytedance-research/UI-TARS-72B-SFT', 'bytedance-research/UI-TARS-72B-SFT'),
+                Model('bytedance-research/UI-TARS-72B-DPO', 'bytedance-research/UI-TARS-72B-DPO'),
+            ]),
+            ModelGroup([
+                Model('allenai/olmOCR-7B-0225-preview', 'allenai/olmOCR-7B-0225-preview'),
+            ]),
+        ],
+        TemplateType.qwen2_vl,
+        get_model_tokenizer_qwen2_vl,
+        model_arch=ModelArch.qwen2_vl,
+        architectures=['Qwen2VLForConditionalGeneration'],
+        requires=['transformers>=4.45', 'qwen_vl_utils>=0.0.6', 'decord'],
+        tags=['vision', 'video']))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qvq, [
+            ModelGroup([
+                Model('Qwen/QVQ-72B-Preview', 'Qwen/QVQ-72B-Preview'),
+            ]),
+        ],
+        TemplateType.qvq,
+        get_model_tokenizer_qwen2_vl,
+        model_arch=ModelArch.qwen2_vl,
+        architectures=['Qwen2VLForConditionalGeneration'],
+        requires=['transformers>=4.45', 'qwen_vl_utils>=0.0.6', 'decord'],
+        tags=['vision', 'video']))
+
+
+def get_model_tokenizer_qwen2_5_vl(*args, **kwargs):
+    from transformers import Qwen2_5_VLForConditionalGeneration
+    kwargs['automodel_class'] = kwargs['automodel_class'] or Qwen2_5_VLForConditionalGeneration
+    return get_model_tokenizer_qwen2_vl(*args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_5_vl, [
+            ModelGroup([
+                Model('Qwen/Qwen2.5-VL-3B-Instruct', 'Qwen/Qwen2.5-VL-3B-Instruct'),
+                Model('Qwen/Qwen2.5-VL-7B-Instruct', 'Qwen/Qwen2.5-VL-7B-Instruct'),
+                Model('Qwen/Qwen2.5-VL-32B-Instruct', 'Qwen/Qwen2.5-VL-32B-Instruct'),
+                Model('Qwen/Qwen2.5-VL-72B-Instruct', 'Qwen/Qwen2.5-VL-72B-Instruct'),
+            ]),
+            ModelGroup([
+                Model('Qwen/Qwen2.5-VL-3B-Instruct-AWQ', 'Qwen/Qwen2.5-VL-3B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-VL-7B-Instruct-AWQ', 'Qwen/Qwen2.5-VL-7B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-VL-32B-Instruct-AWQ', 'Qwen/Qwen2.5-VL-32B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-VL-72B-Instruct-AWQ', 'Qwen/Qwen2.5-VL-72B-Instruct-AWQ'),
+            ]),
+        ],
+        TemplateType.qwen2_5_vl,
+        get_model_tokenizer_qwen2_5_vl,
+        model_arch=ModelArch.qwen2_vl,
+        architectures=['Qwen2_5_VLForConditionalGeneration'],
+        requires=['transformers>=4.49', 'qwen_vl_utils>=0.0.6', 'decord'],
+        tags=['vision', 'video']))
+
+
+def get_model_tokenizer_qwen2_5_omni(model_dir, *args, **kwargs):
+    from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor, Qwen2_5OmniConfig
+    from qwen_omni_utils import vision_process
+    kwargs['automodel_class'] = kwargs['automodel_class'] or Qwen2_5OmniForConditionalGeneration
+    processor = Qwen2_5OmniProcessor.from_pretrained(model_dir, trust_remote_code=True)
+    kwargs['tokenizer'] = processor.tokenizer
+    kwargs['model_config'] = Qwen2_5OmniConfig.from_pretrained(model_dir, trust_remote_code=True)
+    patch_qwen_vl_utils(vision_process)
+    kwargs['model_config'].enable_audio_output = get_env_args('ENABLE_AUDIO_OUTPUT', bool, True)
+    model, _ = get_model_tokenizer_with_flash_attn(model_dir, *args, **kwargs)
+    if model:
+        use_submodel_func(model, 'thinker')
+        model.config.keys_to_ignore_at_inference += ['hidden_states', 'attention_mask']
+        model.config.talker_config.pad_token_id = None
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_5_omni,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2.5-Omni-3B', 'Qwen/Qwen2.5-Omni-3B'),
+                Model('Qwen/Qwen2.5-Omni-7B', 'Qwen/Qwen2.5-Omni-7B'),
+            ]),
+        ],
+        TemplateType.qwen2_5_omni,
+        get_model_tokenizer_qwen2_5_omni,
+        model_arch=ModelArch.qwen2_5_omni,
+        architectures=['Qwen2_5OmniModel'],
+        requires=['transformers>=4.50', 'soundfile', 'qwen_omni_utils', 'decord'],
+        tags=['vision', 'video', 'audio'],
+        additional_saved_files=['spk_dict.pt'],
+        ignore_patterns=[],
+    ))
+
+
+def get_model_tokenizer_qwen2_audio(*args, **kwargs):
+    from transformers import Qwen2AudioForConditionalGeneration
+    kwargs['automodel_class'] = kwargs['automodel_class'] or Qwen2AudioForConditionalGeneration
+    return get_model_tokenizer_multimodal(*args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_audio,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2-Audio-7B-Instruct', 'Qwen/Qwen2-Audio-7B-Instruct'),
+                Model('Qwen/Qwen2-Audio-7B', 'Qwen/Qwen2-Audio-7B'),
+            ]),
+        ],
+        TemplateType.qwen2_audio,
+        get_model_tokenizer_qwen2_audio,
+        model_arch=ModelArch.qwen2_audio,
+        architectures=['Qwen2AudioForConditionalGeneration'],
+        requires=['transformers>=4.45,<4.49', 'librosa'],
+        tags=['audio'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.marco_o1, [ModelGroup([Model('AIDC-AI/Marco-o1', 'AIDC-AI/Marco-o1')])],
+        TemplateType.marco_o1,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwq_preview, [ModelGroup([Model('Qwen/QwQ-32B-Preview', 'Qwen/QwQ-32B-Preview')])],
+        TemplateType.qwq_preview,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwq,
+        [ModelGroup([
+            Model('Qwen/QwQ-32B', 'Qwen/QwQ-32B'),
+            Model('Qwen/QwQ-32B-AWQ', 'Qwen/QwQ-32B-AWQ'),
+        ])],
+        TemplateType.qwq,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37']))
+
+
+def get_model_tokenizer_ovis(*args, **kwargs):
+    kwargs['attn_impl_keys'] = ['llm_attn_implementation']
+    model, tokenizer = get_model_tokenizer_with_flash_attn(*args, **kwargs)
+    if model is not None:
+        model.visual_tokenizer.to(model.dtype)
+        model.vte.to(model.dtype)
+
+        model.generation_config.cache_implementation = None
+        func_list = ['generate', 'forward', 'get_input_embeddings']
+        use_submodel_func(model, 'llm', func_list)
+        embedding = model.get_input_embeddings()
+        patch_output_clone(embedding)
+    try:
+        # fix device_map
+        from transformers.cache_utils import HybridCache
+
+        def update(self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx: int, *args,
+                   **kwargs) -> Tuple[torch.Tensor]:
+            self.key_cache[layer_idx] = self.key_cache[layer_idx].to(key_states.device)
+            self.value_cache[layer_idx] = self.value_cache[layer_idx].to(value_states.device)
+            return self._update_origin(key_states, value_states, layer_idx, *args, **kwargs)
+
+        if not hasattr(HybridCache, '_update_origin'):
+            HybridCache._update_origin = HybridCache.update
+            HybridCache.update = update
+    except ImportError:
+        pass
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.ovis1_6,
+        [
+            ModelGroup([
+                Model('AIDC-AI/Ovis1.6-Gemma2-9B', 'AIDC-AI/Ovis1.6-Gemma2-9B'),
+                Model('AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4', 'AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4'),
+                Model('AIDC-AI/Ovis1.6-Gemma2-27B', 'AIDC-AI/Ovis1.6-Gemma2-27B'),
+            ]),
+        ],
+        TemplateType.ovis1_6,
+        get_model_tokenizer_ovis,
+        model_arch=ModelArch.ovis1_6,
+        architectures=['Ovis'],
+        tags=['vision'],
+        requires=['transformers>=4.42'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.ovis1_6_llama3,
+        [
+            ModelGroup([
+                Model('AIDC-AI/Ovis1.6-Llama3.2-3B', 'AIDC-AI/Ovis1.6-Llama3.2-3B'),
+            ]),
+        ],
+        TemplateType.ovis1_6_llama3,
+        get_model_tokenizer_ovis,
+        model_arch=ModelArch.ovis1_6,
+        architectures=['Ovis'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.ovis2,
+        [
+            ModelGroup([
+                Model('AIDC-AI/Ovis2-1B', 'AIDC-AI/Ovis2-1B'),
+                Model('AIDC-AI/Ovis2-2B', 'AIDC-AI/Ovis2-2B'),
+                Model('AIDC-AI/Ovis2-4B', 'AIDC-AI/Ovis2-4B'),
+                Model('AIDC-AI/Ovis2-8B', 'AIDC-AI/Ovis2-8B'),
+                Model('AIDC-AI/Ovis2-16B', 'AIDC-AI/Ovis2-16B'),
+                Model('AIDC-AI/Ovis2-34B', 'AIDC-AI/Ovis2-34B'),
+            ]),
+        ],
+        TemplateType.ovis2,
+        get_model_tokenizer_ovis,
+        model_arch=ModelArch.ovis1_6,
+        architectures=['Ovis'],
+        tags=['vision'],
+        requires=['transformers>=4.46.2', 'moviepy<2'],
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.qwen2_reward,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2-Math-RM-72B', 'Qwen/Qwen2-Math-RM-72B'),
+            ]),
+        ],
+        TemplateType.qwen,
+        get_model_tokenizer_reward_model,
+        architectures=['Qwen2ForRewardModel'],
+        requires=['transformers>=4.37'],
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.qwen2_5_prm,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2.5-Math-PRM-7B', 'Qwen/Qwen2.5-Math-PRM-7B'),
+                Model('Qwen/Qwen2.5-Math-7B-PRM800K', 'Qwen/Qwen2.5-Math-7B-PRM800K'),
+                Model('Qwen/Qwen2.5-Math-PRM-72B', 'Qwen/Qwen2.5-Math-PRM-72B'),
+            ]),
+        ],
+        TemplateType.qwen2_5_math_prm,
+        get_model_tokenizer_reward_model,
+        task_type='prm',
+        architectures=['Qwen2ForProcessRewardModel'],
+        requires=['transformers>=4.37'],
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.qwen2_5_math_reward,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2.5-Math-RM-72B', 'Qwen/Qwen2.5-Math-RM-72B'),
+            ]),
+        ],
+        TemplateType.qwen2_5_math,
+        get_model_tokenizer_reward_model,
+        architectures=['Qwen2ForRewardModel'],
+        requires=['transformers>=4.37'],
+    ))

ms-swift/swift/llm/model/model/skywork.py

@@ -0,0 +1,89 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict
+
+from swift.llm import TemplateType
+from ..constant import LLMModelType, RMModelType
+from ..model_arch import ModelArch
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import ModelInfo
+
+
+def get_skywork_model_tokenizer(model_dir: str,
+                                model_info: ModelInfo,
+                                model_kwargs: Dict[str, Any],
+                                load_model: bool = True,
+                                **kwargs):
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if 'chat' in model_dir:
+        tokenizer.add_tokens('[USER]')
+        tokenizer.add_tokens('[BOT]')
+        tokenizer.add_tokens('[SEP]')
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.skywork,
+        [
+            ModelGroup([
+                Model('skywork/Skywork-13B-base', 'skywork/Skywork-13B-base'),
+                Model('skywork/Skywork-13B-chat'),
+            ]),
+        ],
+        TemplateType.skywork,
+        get_skywork_model_tokenizer,
+        architectures=['SkyworkForCausalLM'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.skywork_o1,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Skywork-o1-Open-Llama-3.1-8B', 'Skywork/Skywork-o1-Open-Llama-3.1-8B'),
+            ]),
+        ],
+        TemplateType.skywork_o1,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['LlamaForCausalLM'],
+        requires=['transformers>=4.43'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.llama3_2_reward,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Skywork-Reward-Llama-3.1-8B', 'Skywork/Skywork-Reward-Llama-3.1-8B'),
+                Model('AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2', 'Skywork/Skywork-Reward-Llama-3.1-8B-v0.2'),
+            ]),
+            ModelGroup([
+                Model('AI-ModelScope/GRM_Llama3.1_8B_rewardmodel-ft', 'Ray2333/GRM_Llama3.1_8B_rewardmodel-ft'),
+                Model('AI-ModelScope/GRM-llama3.2-3B-rewardmodel-ft', 'Ray2333/GRM-llama3.2-3B-rewardmodel-ft'),
+            ])
+        ],
+        TemplateType.llama3_2,
+        get_model_tokenizer_with_flash_attn,
+        requires=['transformers>=4.43'],
+        architectures=['LlamaForSequenceClassification'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.gemma_reward,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Skywork-Reward-Gemma-2-27B', 'Skywork/Skywork-Reward-Gemma-2-27B'),
+                Model('AI-ModelScope/Skywork-Reward-Gemma-2-27B-v0.2', 'Skywork/Skywork-Reward-Gemma-2-27B-v0.2'),
+            ]),
+        ],
+        TemplateType.gemma,
+        get_model_tokenizer_with_flash_attn,
+        requires=['transformers>=4.42'],
+        architectures=['Gemma2ForSequenceClassification'],
+        model_arch=ModelArch.llama,
+    ))

ms-swift/swift/llm/model/model/yi.py

@@ -0,0 +1,146 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import sys
+from typing import Any, Dict
+
+from transformers import AutoTokenizer
+
+from swift.llm import TemplateType
+from swift.utils import get_logger
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import ModelInfo, git_clone_github
+
+logger = get_logger()
+
+
+def get_model_tokenizer_yi(model_dir, *args, **kwargs):
+    tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, use_fast=False)
+    return get_model_tokenizer_with_flash_attn(model_dir, *args, tokenizer=tokenizer, **kwargs)
+
+
+def get_model_tokenizer_yi_vl(model_dir: str,
+                              model_info: ModelInfo,
+                              model_kwargs: Dict[str, Any],
+                              load_model: bool = True,
+                              **kwargs):
+    local_repo_path = kwargs.get('local_repo_path')
+    if not local_repo_path:
+        local_repo_path = git_clone_github('https://github.com/01-ai/Yi')
+    sys.path.append(os.path.join(local_repo_path, 'VL'))
+    from llava.model import LlavaLlamaForCausalLM, LlavaConfig
+    from llava.model.constants import key_info
+
+    model_config = LlavaConfig.from_pretrained(model_dir)
+    mm_vision_tower = model_config.mm_vision_tower
+    model_config.mm_vision_tower = os.path.join(model_dir, *mm_vision_tower.rsplit('/', maxsplit=2)[-2:])
+    model_config.attention_dropout = 0.
+    key_info['model_path'] = model_dir
+    kwargs['automodel_class'] = kwargs['automodel_class'] or LlavaLlamaForCausalLM
+    model, tokenizer = get_model_tokenizer_yi(
+        model_dir, model_info, model_kwargs, load_model, model_config=model_config, **kwargs)
+    if model is not None:
+        logger.info('Please ignore the above warning.')
+        logger.info('Loading the parameters of vision_tower...')
+        model.resize_token_embeddings(len(tokenizer))
+        vision_tower = model.get_vision_tower()
+        vision_tower.load_model()
+        vision_tower.to(device=model.device, dtype=model_config.torch_dtype)
+        if not hasattr(model.config, 'max_sequence_length'):
+            model.config.max_sequence_length = 2048
+        tokenizer.image_processor = vision_tower.image_processor
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.yi_vl,
+        [
+            ModelGroup([
+                Model('01ai/Yi-VL-6B', '01-ai/Yi-VL-6B'),
+                Model('01ai/Yi-VL-34B', '01-ai/Yi-VL-34B'),
+            ], ),
+        ],
+        TemplateType.yi_vl,
+        get_model_tokenizer_yi_vl,
+        model_arch=ModelArch.llava_llama,
+        architectures=['LlavaLlamaForCausalLM'],
+        requires=['transformers>=4.34'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.yi,
+        [  # yi
+            ModelGroup([
+                Model('01ai/Yi-6B', '01-ai/Yi-6B'),
+                Model('01ai/Yi-6B-200K', '01-ai/Yi-6B-200K'),
+                Model('01ai/Yi-6B-Chat', '01-ai/Yi-6B-Chat'),
+                Model('01ai/Yi-6B-Chat-4bits', '01-ai/Yi-6B-Chat-4bits'),
+                Model('01ai/Yi-6B-Chat-8bits', '01-ai/Yi-6B-Chat-8bits'),
+                Model('01ai/Yi-9B', '01-ai/Yi-9B'),
+                Model('01ai/Yi-9B-200K', '01-ai/Yi-9B-200K'),
+                Model('01ai/Yi-34B', '01-ai/Yi-34B'),
+                Model('01ai/Yi-34B-200K', '01-ai/Yi-34B-200K'),
+                Model('01ai/Yi-34B-Chat', '01-ai/Yi-34B-Chat'),
+                Model('01ai/Yi-34B-Chat-4bits', '01-ai/Yi-34B-Chat-4bits'),
+                Model('01ai/Yi-34B-Chat-8bits', '01-ai/Yi-34B-Chat-8bits'),
+            ]),
+            # yi1.5
+            ModelGroup([
+                Model('01ai/Yi-1.5-6B', '01-ai/Yi-1.5-6B'),
+                Model('01ai/Yi-1.5-6B-Chat', '01-ai/Yi-1.5-6B-Chat'),
+                Model('01ai/Yi-1.5-9B', '01-ai/Yi-1.5-9B'),
+                Model('01ai/Yi-1.5-9B-Chat', '01-ai/Yi-1.5-9B-Chat'),
+                Model('01ai/Yi-1.5-9B-Chat-16K', '01-ai/Yi-1.5-9B-Chat-16K'),
+                Model('01ai/Yi-1.5-34B', '01-ai/Yi-1.5-34B'),
+                Model('01ai/Yi-1.5-34B-Chat', '01-ai/Yi-1.5-34B-Chat'),
+                Model('01ai/Yi-1.5-34B-Chat-16K', '01-ai/Yi-1.5-34B-Chat-16K'),
+            ]),
+            # yi1.5-quant
+            ModelGroup([
+                Model('AI-ModelScope/Yi-1.5-6B-Chat-GPTQ', 'modelscope/Yi-1.5-6B-Chat-GPTQ'),
+                Model('AI-ModelScope/Yi-1.5-6B-Chat-AWQ', 'modelscope/Yi-1.5-6B-Chat-AWQ'),
+                Model('AI-ModelScope/Yi-1.5-9B-Chat-GPTQ', 'modelscope/Yi-1.5-9B-Chat-GPTQ'),
+                Model('AI-ModelScope/Yi-1.5-9B-Chat-AWQ', 'modelscope/Yi-1.5-9B-Chat-AWQ'),
+                Model('AI-ModelScope/Yi-1.5-34B-Chat-GPTQ', 'modelscope/Yi-1.5-34B-Chat-GPTQ'),
+                Model('AI-ModelScope/Yi-1.5-34B-Chat-AWQ', 'modelscope/Yi-1.5-34B-Chat-AWQ'),
+            ]),
+        ],
+        TemplateType.chatml,
+        get_model_tokenizer_yi,
+        architectures=['LlamaForCausalLM'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.yi_coder, [
+            ModelGroup([
+                Model('01ai/Yi-Coder-1.5B', '01-ai/Yi-Coder-1.5B'),
+                Model('01ai/Yi-Coder-9B', '01-ai/Yi-Coder-9B'),
+                Model('01ai/Yi-Coder-1.5B-Chat', '01-ai/Yi-Coder-1.5B-Chat'),
+                Model('01ai/Yi-Coder-9B-Chat', '01-ai/Yi-Coder-9B-Chat'),
+            ], )
+        ],
+        TemplateType.yi_coder,
+        get_model_tokenizer_yi,
+        architectures=['LlamaForCausalLM'],
+        model_arch=ModelArch.llama,
+        tags=['coding']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.sus,
+        [
+            ModelGroup([
+                Model('SUSTC/SUS-Chat-34B', 'SUSTech/SUS-Chat-34B'),
+            ]),
+        ],
+        TemplateType.sus,
+        get_model_tokenizer_yi,
+        model_arch=ModelArch.llama,
+        architectures=['LlamaForCausalLM'],
+    ))

ms-swift/swift/llm/sampling/__init__.py

@@ -0,0 +1 @@
+from .sampling import sampling_main

ms-swift/swift/llm/sampling/base.py

@@ -0,0 +1,55 @@
+from typing import Any, Dict, List
+
+from swift.llm import SamplingArguments
+from swift.plugin import orms, prms
+from swift.utils import get_logger
+
+logger = get_logger()
+
+
+class Sampler:
+
+    def __init__(self, input_args: SamplingArguments):
+        self.args = input_args
+        self.template = None
+        self.processor = None
+        self.prm_model = None
+        self.orm_model = None
+        self._prepare_model_tokenizer()
+        self._prepare_template()
+        self._prepare_rm()
+
+    def _prepare_model_tokenizer(self):
+        args = self.args
+        _, self.processor = args.get_model_processor(load_model=False)
+
+    def _prepare_rm(self):
+        if self.args.prm_model is None:
+            self.prm_model = None
+            logger.warning('prm_model is None.')
+        elif self.args.prm_model in prms:
+            self.prm_model = prms[self.args.prm_model]()
+        else:
+            from swift.llm import PtEngine
+            self.prm_model = PtEngine(self.args.prm_model, max_batch_size=64)
+
+        if self.args.orm_model is None:
+            self.orm_model = None
+            logger.warning('orm_model is None.')
+        elif self.args.orm_model in orms:
+            self.orm_model = orms[self.args.orm_model]()
+        else:
+            from swift.llm import PtEngine
+            self.orm_model = PtEngine(self.args.orm_model, max_batch_size=64)
+
+    def _prepare_template(self) -> None:
+        template = self.args.get_template(self.processor)
+        self.template = template
+        self.template.set_mode('train')
+
+    def truncate_input(self, slices: List[Dict[str, Any]]):
+        """Truncate the input rows to avoid hitting the max length of the policy model"""
+        return slices
+
+    def do_sample(self, data):
+        raise NotImplementedError

ms-swift/swift/llm/sampling/sampling.py

@@ -0,0 +1,106 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import shutil
+import time
+from typing import List, Union
+
+import json
+
+from swift.llm import SamplingArguments, SwiftPipeline, load_dataset
+from swift.utils import get_logger
+
+logger = get_logger()
+
+
+class SwiftSampling(SwiftPipeline):
+    args_class = SamplingArguments
+    args: args_class
+
+    def __init__(self, args: Union[List[str], SamplingArguments, None] = None) -> None:
+        super().__init__(args)
+        self.args.save_args()
+        os.makedirs(self.args.output_dir, exist_ok=True)
+        self.cur_piece = 0
+        self.total_piece = 1
+
+        if self.args.data_range:
+            self.cur_piece, self.total_piece = self.args.data_range
+
+        if self.args.sampler_type == 'sample':
+            from swift.llm.sampling.vanilla_sampler import VanillaSampler
+            self.sampler = VanillaSampler(self.args)
+        elif self.args.sampler_type == 'mcts':
+            from swift.llm.sampling.mcts import MctsSampler
+            self.sampler = MctsSampler(self.args)
+        elif self.args.sampler_type == 'distill':
+            from swift.llm.sampling.distill_sampler import DistillSampler
+            self.sampler = DistillSampler(self.args)
+        else:
+            raise ValueError(f'Unsupported sampler type: {self.args.sampler_type}')
+
+    def _get_dataset(self):
+        args = self.args
+        dataset_kwargs = args.get_dataset_kwargs()
+        sampling_dataset, _ = load_dataset(
+            args.dataset, split_dataset_ratio=0., shuffle=args.dataset_shuffle, **dataset_kwargs)
+        logger.info(f'Sampling_dataset: {sampling_dataset}')
+        dataset_len = len(sampling_dataset)
+        piece_len = dataset_len // self.total_piece
+        sampling_dataset = sampling_dataset.select(range(piece_len * self.cur_piece, piece_len * (self.cur_piece + 1)))
+        return sampling_dataset
+
+    def run(self):
+        os.makedirs(self.args.output_dir, exist_ok=True)
+        iter_file = os.path.join(self.args.output_dir, self.args.output_file)
+        resume_file = os.path.join(self.args.output_dir, self.args.output_file + '.resume')
+        tmp_file = os.path.join(self.args.output_dir, self.args.output_file + '.tmp')
+        ckpt_state_file = os.path.join(self.args.output_dir, 'ckpt_state.json')
+        if os.path.exists(iter_file) and not self.args.override_exist_file:
+            return
+
+        index_resume = -1
+        write_mode = 'w'
+        if self.args.resume:
+            write_mode = 'a'
+            if os.path.exists(resume_file):
+                shutil.copyfile(resume_file, tmp_file)
+
+            if os.path.exists(ckpt_state_file):
+                with open(ckpt_state_file, 'r') as ckpt_state:
+                    data = json.load(ckpt_state)
+                    index_resume = data.get('index', -1)
+                    logger.info(f'Loaded index_resume: {index_resume}')
+        else:
+            if os.path.exists(tmp_file):
+                os.remove(tmp_file)
+
+        dataset = self._get_dataset()
+        dataset_len = len(dataset)
+        total_iters = int(dataset_len // self.args.num_sampling_per_gpu_batch_size)
+
+        if self.args.num_sampling_per_gpu_batches is None or self.args.num_sampling_per_gpu_batches > total_iters:
+            self.args.num_sampling_per_gpu_batches = total_iters
+
+        with open(tmp_file, write_mode) as f:
+            for _index in range(self.args.num_sampling_per_gpu_batches):
+                if _index <= index_resume:
+                    continue
+                logger.info(f' Sampling index:{_index}')
+                slices = dataset[self.args.num_sampling_per_gpu_batch_size
+                                 * _index:self.args.num_sampling_per_gpu_batch_size * (_index + 1)]
+                slices = self.sampler.truncate_input(slices)
+                generated = self.sampler.do_sample(slices)
+                f.writelines(generated)
+                f.flush()
+                shutil.copy(tmp_file, resume_file)
+                with open(ckpt_state_file, 'w') as ckpt_state:
+                    json.dump({'index': _index}, ckpt_state)
+
+        if os.path.exists(iter_file):
+            shutil.move(iter_file, iter_file + '.' + str(int(time.time())))
+        shutil.move(resume_file, iter_file)
+        logger.info(f'Sample file {iter_file} generated.')
+
+
+def sampling_main(args: Union[List[str], SamplingArguments, None] = None):
+    return SwiftSampling(args).main()

ms-swift/swift/llm/sampling/utils.py

@@ -0,0 +1,172 @@
+import hashlib
+import inspect
+from copy import copy
+from typing import Any, Dict, List, Optional
+
+import json
+import numpy as np
+
+from swift.llm import InferRequest, RequestConfig
+from swift.utils import get_logger
+
+logger = get_logger()
+
+
+def get_messages_md5(row: Dict[str, Any]):
+    row = copy(row)
+    row.pop('choices', None)
+    serialized = json.dumps(row, sort_keys=True)
+    return hashlib.md5(serialized.encode('utf-8')).hexdigest()
+
+
+def get_reward(model: Any,
+               infer_requests: List[InferRequest],
+               request_config: RequestConfig = None,
+               ground_truths: List[str] = None,
+               threshold: Optional[float] = None):
+    """Get reward from an RM model.
+
+    Args:
+        model: The model instance or an RM evaluator
+        infer_requests: Infer requests sent to the model
+        request_config: Infer config
+        ground_truths: The ground truth list
+        threshold: An optional threshold to generate the mask
+
+    Returns:
+        Tuple
+        Index 0: The min-max normalized scores matched the infer_requests
+        Index 1: The mask filtered by the threshold
+    """
+    from swift.llm import InferEngine
+    infer_func = model.infer if isinstance(model, InferEngine) else model.__call__
+    parameters = inspect.signature(infer_func).parameters
+    gt_param = {}
+    if 'ground_truths' in parameters:
+        gt_param = {'ground_truths': ground_truths}
+    if isinstance(infer_requests[0], dict):
+        infer_requests = [InferRequest(messages=req['messages']) for req in infer_requests]
+    rewards = infer_func(infer_requests, request_config=request_config, **gt_param)
+    from swift.llm.infer.protocol import ChatCompletionResponse
+    if isinstance(rewards[0], ChatCompletionResponse):
+        print('reward:', rewards[0].choices[0].message.content)
+        if isinstance(rewards[0].choices[0].message.content, str):
+            rewards = [float(r.choices[0].message.content.strip('[]')) for r in rewards]
+        elif isinstance(rewards[0].choices[0].message.content, list):
+            rewards = [float(min(r.choices[0].message.content)) for r in rewards]
+        else:
+            rewards = [float(r.choices[0].message.content) for r in rewards]
+    arr = []
+    for reward in rewards:
+        if isinstance(reward, (list, tuple)):
+            arr.append(min(reward))
+        else:
+            arr.append(float(reward))
+
+    _mask = np.array([True] * len(arr))
+    if threshold is not None:
+        # > not >=, orm caller passes 0, which will cause error
+        _mask = np.array([a > threshold for a in arr])
+
+    def normalize(arr):
+        min_val = np.min(arr)
+        max_val = np.max(arr)
+        if min_val == max_val:
+            if min_val == 0:
+                constant_value = 0.0
+            else:
+                constant_value = min(1.0, min_val)
+            return np.full_like(arr, fill_value=constant_value, dtype=np.float64)
+        normalized = (arr - min_val) / (max_val - min_val + 1e-5)
+        return normalized
+
+    return normalize(arr), _mask
+
+
+def perform_infer(infer_engines, infer_requests, request_configs, **infer_kwargs):
+    if isinstance(infer_engines, list):
+        assert len(infer_engines) >= len(request_configs) >= len(infer_requests)
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+        n = len(infer_requests)
+        with ThreadPoolExecutor(max_workers=n) as executor:
+            futures = {
+                executor.submit(perform_infer, infer_engines[i], infer_requests[i], request_configs[i], **infer_kwargs):
+                i
+                for i in range(n)
+            }
+            responses = []
+            for future in as_completed(futures):
+                task_id = futures[future]
+                try:
+                    responses += future.result()
+                except Exception as e:
+                    logger.info(f'Perform infer task: {task_id} get an error: {e}')
+        return responses
+    elif isinstance(infer_requests, list):
+        responses = []
+        if isinstance(request_configs, list):
+            assert len(infer_requests) <= len(request_configs)
+            for i in range(len(infer_requests)):
+                responses += infer_engines.infer(
+                    [infer_requests[i]],
+                    request_configs[i],
+                    **infer_kwargs,
+                )
+        elif isinstance(request_configs, RequestConfig):
+            for infer_request in infer_requests:
+                responses += infer_engines.infer(
+                    [infer_request],
+                    request_configs,
+                    **infer_kwargs,
+                )
+        return responses
+    return infer_engines.infer(
+        [infer_requests],
+        request_configs,
+        **infer_kwargs,
+    )
+
+
+def collect_from_mct(monte_carlo_tree, collect_filter_threshold):
+    from transformers.utils import strtobool
+    if isinstance(monte_carlo_tree, str):
+        monte_carlo_tree = json.loads(monte_carlo_tree)
+
+    def _collect(collect_curr_node, _outcome_rewards: list[float], _process_rewards: list[float]):
+        _prefer_pairs, _correct_answers, _incorrect_answers = [], [], []
+        _outcome_rewards = _outcome_rewards[:] + [collect_curr_node['outcome_reward']]
+        _process_rewards = _process_rewards[:] + [collect_curr_node['process_reward']]
+        if len(collect_curr_node['children']) > 0:
+            for child in collect_curr_node['children']:
+                p, c, i = _collect(child, _outcome_rewards, _process_rewards)
+                _prefer_pairs += p
+                _correct_answers += c
+                _incorrect_answers += i
+            sorted_children = sorted(collect_curr_node['children'], key=lambda x: x['outcome_reward'])
+            if sorted_children[-1]['outcome_reward'] - sorted_children[0]['outcome_reward'] > collect_filter_threshold:
+                # TODO: filter with visit count
+                prefer_pair = {
+                    'path': 'ки\n'.join(collect_curr_node['path']),
+                    'good': sorted_children[-1]['path'][-1],
+                    'good_score': sorted_children[-1]['outcome_reward'],
+                    'bad': sorted_children[0]['path'][-1],
+                    'bad_score': sorted_children[0]['outcome_reward'],
+                }
+                _prefer_pairs.append(prefer_pair)
+        if strtobool(collect_curr_node['terminated']):
+            _answer = {
+                'answer': 'ки\n'.join(collect_curr_node['path']),
+                'mean_outcome_reward': np.mean(_outcome_rewards),
+                'min_outcome_reward': np.min(_outcome_rewards),
+                'mean_process_reward': np.mean(_process_rewards),
+                'min_process_reward': np.min(_process_rewards),
+            }
+            if strtobool(collect_curr_node['correct']):
+                _correct_answers.append(_answer)
+            else:
+                _incorrect_answers.append(_answer)
+        return _prefer_pairs, _correct_answers, _incorrect_answers
+
+    _root = monte_carlo_tree
+    prefer_pairs, correct_answers, incorrect_answers = _collect(_root, [], [])
+    return prefer_pairs, correct_answers, incorrect_answers

ms-swift/swift/llm/sampling/vanilla_sampler.py

@@ -0,0 +1,216 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from copy import deepcopy
+from typing import Any, Dict, List
+
+import json
+import numpy as np
+
+from swift.llm import RequestConfig
+from swift.llm.sampling.base import Sampler
+from swift.llm.template.template_inputs import InferRequest
+from swift.utils import get_logger
+from .utils import get_messages_md5, get_reward
+
+logger = get_logger()
+
+
+class VanillaSampler(Sampler):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if self.args.sampler_engine == 'pt':
+            from swift.llm import PtEngine
+            _Engine = PtEngine
+        elif self.args.sampler_engine == 'vllm':
+            from swift.llm import VllmEngine
+            _Engine = VllmEngine
+        elif self.args.sampler_engine == 'lmdeploy':
+            from swift.llm import LmdeployEngine
+            _Engine = LmdeployEngine
+        elif self.args.sampler_engine == 'no':
+            _Engine = None
+        else:
+            raise ValueError(f'Cannot find engine name: {self.args.sampler_engine}')
+        self.infer_engine = None
+        if _Engine:
+            self.infer_engine = _Engine(self.args.model, model_type=self.args.model_type, **self.args.engine_kwargs)
+            self.infer_engine.default_template = self.template
+            self.infer_engine.strict = False
+        self.caches = self.read_cache()
+
+    def read_cache(self):
+        cache_files = self.args.cache_files
+        caches = {}
+        for file in cache_files:
+            if not os.path.exists(file):
+                logger.warning(f'Cache file does not exist: {file}')
+                continue
+            with open(file, 'r') as f:
+                for line in f.readlines():
+                    line = line.strip()
+                    if not line:
+                        continue
+
+                    content = json.loads(line)
+                    uuid = content['id']
+                    messages = content['messages']
+                    if uuid not in caches:
+                        caches[uuid] = {'choices': []}
+                    assert messages[-1]['role'] == 'assistant'
+                    caches[uuid]['choices'].append(messages[-1]['content'])
+        return caches
+
+    @staticmethod
+    def convert_data_to_rows(data):
+        rows = []
+        key = list(data.keys())[0]
+        data_len = len(data[key])
+        for idx in range(data_len):
+            row = {key: data[key][idx] for key in data}
+            if row.get('images') and 'bytes' in row['images'][0]:
+                row['images'] = [img['path'] for img in row['images']]
+            rows.append(row)
+        VanillaSampler.check_row_valid(rows)
+        return rows
+
+    @staticmethod
+    def check_row_valid(rows):
+        for row in rows:
+            assert not row.get('images') or all([isinstance(img, str) and img for img in row['images']])
+            assert not row.get('videos') or all([isinstance(video, str) and video for video in row['videos']])
+            assert not row.get('audios') or all([isinstance(audio, str) and audio for audio in row['audios']])
+
+    def generate(self, data):
+        resp_all = []
+        infer_requests = []
+        sent = 0
+        rows = self.convert_data_to_rows(data)
+        for idx, row in enumerate(rows):
+            row = deepcopy(row)
+            messages = row['messages']
+            uuid = get_messages_md5(row)
+            if uuid in self.caches:
+                choices = self.caches[uuid]['choices']
+                if len(choices) == self.args.num_return_sequences:
+                    continue
+            if self.args.system:
+                if messages[0]['role'] == 'system':
+                    messages[0]['content'] = self.args.system
+                else:
+                    messages.insert(0, {'role': 'system', 'content': self.args.system})
+            if messages[-1]['role'] == 'assistant':
+                messages = messages[:-1]
+
+            row['messages'] = messages
+            infer_request = row
+            for i in range(self.args.num_return_sequences):
+                infer_requests.append(deepcopy(infer_request))
+            sent += 1
+
+        request_config = RequestConfig(
+            max_tokens=self.args.max_new_tokens,
+            temperature=self.args.temperature,
+            top_k=self.args.top_k,
+            top_p=self.args.top_p,
+        )
+
+        resp_list = []
+        if len(infer_requests) > 0:
+            resp_list = self.infer_engine.infer(infer_requests, request_config=request_config)
+
+        _cur = 0
+        for idx, row in enumerate(rows):
+            row = deepcopy(row)
+            uuid = get_messages_md5(row)
+            if uuid in self.caches:
+                choices = self.caches[uuid]['choices']
+                if len(choices) == self.args.num_return_sequences:
+                    row['choices'] = choices
+                    resp_all.append(row)
+                    continue
+
+            resps = row
+            resps['choices'] = []
+            for j in range(self.args.num_return_sequences * _cur, self.args.num_return_sequences * (_cur + 1)):
+                if not isinstance(resp_list[j], Exception):
+                    resps['choices'].append(resp_list[j].choices[0].message.content)
+            if resps['choices']:
+                resp_all.append(resps)
+            _cur += 1
+        return resp_all
+
+    def do_sample(self, data):
+        generated = []
+        resp_all = self.generate(data)
+        for i, resps in enumerate(resp_all):
+            choices = resps['choices']
+            messages = resps['messages']
+            uuid = get_messages_md5(resps)
+            assert messages[-1]['role'] == 'assistant'
+            ground_truth = messages[-1]['content']
+
+            infer_requests = []
+            for decoded in choices:
+                _resps = deepcopy(resps)
+                _resps['messages'][-1]['content'] = decoded
+                infer_requests.append(_resps)
+
+            _resps = deepcopy(resps)
+            _resps['messages'][-1]['content'] = ground_truth
+            infer_requests.append(_resps)
+            if self.orm_model is not None:
+                orm_score, _orm_mask = get_reward(
+                    self.orm_model, infer_requests, ground_truths=[ground_truth] * len(infer_requests), threshold=0.0)
+            else:
+                orm_score = np.array([1.0] * len(infer_requests))
+                _orm_mask = np.array([True] * len(infer_requests))
+            if self.prm_model is not None:
+                prm_score, _prm_mask = get_reward(
+                    self.prm_model,
+                    infer_requests,
+                    ground_truths=[ground_truth] * len(infer_requests),
+                    threshold=self.args.prm_threshold)
+            else:
+                prm_score = np.array([1.0] * len(infer_requests))
+                _prm_mask = np.array([True] * len(infer_requests))
+
+            _mask = _orm_mask & _prm_mask
+            if not any(_mask):
+                continue
+
+            choices.append(ground_truth)
+            choices = np.array(choices)
+
+            if self.orm_model is None and self.prm_model is None:
+                positives = choices[:-1]
+                for positive in positives:
+                    _resps = deepcopy(resps)
+                    _resps.pop('choices', None)
+                    _resps['id'] = uuid
+                    _resps['messages'][-1]['content'] = str(positive)
+                    generated.append(json.dumps(_resps, ensure_ascii=False) + '\n')
+            else:
+                score = np.array(prm_score) + np.array(orm_score * 10)
+                sorted_indices = np.argsort(score)[::-1]
+                pos_indexes = sorted_indices[0:self.args.n_best_to_keep]
+                pos_indexes = [i for i in pos_indexes if _mask[i]]
+                neg_index = sorted_indices[-1]
+                logger.info(
+                    f'orm:{orm_score}, prm:{prm_score}, positive index: {pos_indexes}, negative index: {neg_index}')
+                if self.args.easy_query_threshold is not None and sum([score > 0 for score in orm_score]) - 1 >= int(
+                        self.args.num_return_sequences * self.args.easy_query_threshold):
+                    continue
+                if len(pos_indexes) > 0:
+                    positives = choices[pos_indexes]
+                    negative = choices[neg_index]
+                    for positive in positives:
+                        _resps = deepcopy(resps)
+                        messages = deepcopy(messages)
+                        _resps.pop('choices', None)
+                        _resps['messages'][-1]['content'] = str(positive)
+                        _resps['rejected_response'] = str(negative)
+                        _resps['id'] = uuid
+                        generated.append(json.dumps(_resps, ensure_ascii=False) + '\n')
+        return generated

ms-swift/swift/llm/template/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from . import template
+from .base import MaxLengthError, Template
+from .constant import TemplateType
+from .grounding import draw_bbox
+from .register import TEMPLATE_MAPPING, get_template, get_template_meta, register_template
+from .template_inputs import InferRequest, TemplateInputs
+from .template_meta import TemplateMeta
+from .utils import Prompt, Word, split_str_parts_by
+from .vision_utils import load_file, load_image

ms-swift/swift/llm/template/base.py

@@ -0,0 +1,1606 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import hashlib
+import inspect
+import math
+import os
+import re
+from contextlib import contextmanager, nullcontext
+from copy import deepcopy
+from dataclasses import asdict
+from functools import partial, wraps
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from modelscope.hub.utils.utils import get_cache_dir
+from peft import PeftModel
+from PIL import Image
+from torch.nn.utils.rnn import pad_sequence
+from transformers import StoppingCriteriaList
+from transformers.integrations import is_deepspeed_zero3_enabled
+from transformers.utils import strtobool
+
+from swift.utils import get_dist_setting, get_env_args, get_logger, use_torchacc
+from ..utils import Processor, ProcessorMixin
+from .template_inputs import InferRequest, StdTemplateInputs, TemplateInputs
+from .utils import Context, ContextType, StopWordsCriteria, fetch_one, findall, split_str_parts_by
+from .vision_utils import load_audio, load_batch, load_image, rescale_image
+
+logger = get_logger()
+if TYPE_CHECKING:
+    from .template_meta import TemplateMeta
+
+
+class MaxLengthError(ValueError):
+    pass
+
+
+class Template(ProcessorMixin):
+    special_tokens = ['<image>', '<video>', '<audio>', '<bbox>', '<ref-object>', '<cot-process>', '<start-image>']
+    special_keys = ['images', 'videos', 'audios', 'objects']
+
+    image_placeholder = ['<image>']
+    video_placeholder = ['<video>']
+    audio_placeholder = ['<audio>']
+    cot_process_placeholder = ['ки']
+    placeholder_tokens = []  # For clearer printing
+    load_images = True
+    skip_prompt = True
+    use_model = False
+    norm_bbox = 'norm1000'
+
+    is_encoder_decoder = False
+
+    def __init__(
+        self,
+        processor: Processor,
+        template_meta: 'TemplateMeta',
+        default_system: Optional[str] = None,
+        max_length: Optional[int] = None,
+        *,
+        use_chat_template: bool = True,
+        truncation_strategy: Literal['raise', 'left', 'right'] = 'raise',
+        max_pixels: Optional[int] = None,
+        agent_template: Optional[str] = None,
+        norm_bbox: Literal['norm1000', 'none', None] = None,
+        response_prefix: Optional[str] = None,
+        # only for train
+        padding_side: Literal['left', 'right'] = 'right',
+        loss_scale: str = 'default',
+        sequence_parallel_size: int = 1,
+        # infer/deploy
+        template_backend: Literal['swift', 'jinja'] = 'swift',
+    ) -> None:
+        """
+        default_system: Override the default_system in the template.
+        max_length: Max length of the sequence
+        truncation_strategy: The truncation strategy
+        max_pixels: Rescale image to reduce memory usage, default `None` means no limitation.
+            e.g. 512 * 512 (H*W)
+        padding_side: The padding_side when the training batch_size >= 2
+        loss_scale: The loss scale function to use
+        """
+        from .template_meta import TemplateMeta
+        from swift.plugin import agent_templates
+
+        self.processor = processor
+        self.model_info = processor.model_info
+        self.config = self.model_info.config
+        self.model_meta = processor.model_meta
+        if max_length is None:
+            max_length = self.model_info.max_model_len
+        tokenizer = self.tokenizer
+
+        if not use_chat_template:
+            template_meta = template_meta.to_generate_template_meta()
+        else:
+            template_meta = deepcopy(template_meta)
+        # if default_system is None. not change self.default_system
+        template_meta.check_system(default_system)
+        if default_system is not None:
+            template_meta.default_system = default_system
+        if response_prefix is not None:
+            template_meta.response_prefix = response_prefix
+        logger.info(f'default_system: {repr(template_meta.default_system)}')
+        logger.info(f'response_prefix: {repr(template_meta.response_prefix)}')
+
+        for i, token in enumerate(self.placeholder_tokens):
+            if isinstance(token, str):
+                self.placeholder_tokens[i] = tokenizer.convert_tokens_to_ids(token)
+        template_meta.init(tokenizer)
+
+        self.template_meta: TemplateMeta = template_meta
+        self.use_chat_template = use_chat_template
+        self.template_backend = template_backend
+        self.max_length = max_length
+        self.truncation_strategy = truncation_strategy
+        self.loss_scale = loss_scale
+        self.max_pixels = max_pixels
+        self.padding_side = padding_side
+        self.sequence_parallel_size = sequence_parallel_size
+        agent_template = agent_template or template_meta.agent_template
+        logger.info(f'agent_template: {agent_template}')
+        self.agent_template = agent_templates[agent_template]()
+        self.norm_bbox = norm_bbox or self.norm_bbox
+        logger.info(f'max_length: {self.max_length}')
+        logger.info(f'norm_bbox: {self.norm_bbox}')
+        if self.is_encoder_decoder:
+            self.skip_prompt = False
+        self.mode: Literal['pt', 'vllm', 'lmdeploy',  # infer
+                           'train', 'rlhf', 'kto',  # train
+                           'seq_cls', 'embedding', 'prm'] = 'pt'
+        self._packing = False
+        self.use_megatron = False
+        if self.model_info.task_type != 'causal_lm':
+            self.mode = self.model_info.task_type
+        self._handles = []
+        self._deepspeed_initialize = None
+
+    @staticmethod
+    def _load_image(image, load_images: bool):
+        if load_images:
+            if isinstance(image, dict) and 'bytes' in image:
+                image = image['bytes'] or image['path']
+            image = load_image(image)
+        else:
+            if isinstance(image, dict):
+                path = image['path']
+                if path and (path.startswith('http') or os.path.exists(path)):
+                    image = path
+                else:
+                    image = load_image(image['bytes'])
+            elif not isinstance(image, str):
+                image = load_image(image)
+        return image
+
+    @staticmethod
+    def _get_height_width(inputs: StdTemplateInputs) -> None:
+        width = []
+        height = []
+        for image in inputs.images:
+            width.append(image.width)
+            height.append(image.height)
+        inputs.objects['width'] = width
+        inputs.objects['height'] = height
+
+    def normalize_bbox(self, inputs: StdTemplateInputs) -> None:
+        objects = inputs.objects
+        bbox_list = objects['bbox']
+        width_list = objects['width']
+        height_list = objects['height']
+        bbox_type = objects.pop('bbox_type', None) or 'real'
+        image_id_list = objects.pop('image_id', None) or []
+        image_id_list += [0] * (len(bbox_list) - len(image_id_list))
+        for bbox, image_id in zip(bbox_list, image_id_list):
+            if bbox_type == 'norm1':
+                width, height = 1, 1
+            else:
+                width, height = width_list[image_id], height_list[image_id]
+            for i, (x, y) in enumerate(zip(bbox[::2], bbox[1::2])):
+                if self.norm_bbox == 'norm1000':
+                    norm_width, norm_height = 1000, 1000
+                elif self.norm_bbox == 'none':
+                    image = inputs.images[image_id]
+                    norm_width, norm_height = image.width, image.height
+                bbox[2 * i] = int(round(x / width * norm_width))
+                bbox[2 * i + 1] = int(round(y / height * norm_height))
+
+    def _preprocess_function_call(self, inputs: StdTemplateInputs) -> None:
+        agent_template = self.agent_template
+        agent_template.template_meta = self.template_meta  # for hermes
+        if inputs.tools:
+            if isinstance(inputs.tools, str):
+                inputs.tools = agent_template._parse_json(inputs.tools)
+                if not isinstance(inputs.tools, (list, tuple)):
+                    inputs.tools = [inputs.tools]
+            elif isinstance(inputs.tools, (list, tuple)):
+                inputs.tools = [agent_template._parse_json(tool) for tool in inputs.tools]
+            else:
+                raise ValueError(f'inputs.tools: {inputs.tools}')
+            for i, tool in enumerate(inputs.tools):
+                inputs.tools[i] = agent_template.wrap_tool(tool)
+        i = 0
+        messages = inputs.messages
+        while i < len(messages):
+            if messages[i]['role'] == 'tool_call':
+                i_start = i
+                while i + 1 < len(messages) and messages[i + 1]['role'] == 'tool_call':
+                    i += 1
+                tool_content = self.agent_template._format_tool_calls(messages[i_start:i + 1])
+                messages[i_start:i + 1] = [{'role': 'assistant', 'content': tool_content}]
+                i = i_start + 1
+            else:
+                i += 1
+
+    def _preprocess_inputs(
+        self,
+        inputs: StdTemplateInputs,
+    ) -> None:
+        self._preprocess_function_call(inputs)
+        if self.model_meta.is_multimodal:
+            self._replace_image_tags(inputs)
+            self._replace_start_image_tags(inputs)
+        images = inputs.images
+        load_images = self.load_images or self.mode in {'vllm', 'lmdeploy'}
+        load_images_origin = load_images
+        if self.max_pixels is not None or inputs.objects:
+            load_images = True
+        if images:
+            for i, image in enumerate(images):
+                images[i] = self._load_image(images[i], load_images)
+        if inputs.objects:
+            self._get_height_width(inputs)
+        if self.max_pixels is not None:
+            # Scale the image proportionally without affecting the scaled objects.
+            images = [rescale_image(img, self.max_pixels) for img in images]
+        if images and not load_images_origin:  # fix pt & qwen-vl
+            for i, image in enumerate(images):
+                if isinstance(image, Image.Image):
+                    images[i] = self._save_pil_image(image)
+        inputs.images = images
+
+        if self.mode == 'vllm' and inputs.audios:
+            sampling_rate = get_env_args('sampling_rate', int, None)
+            inputs.audios = load_batch(
+                inputs.audios, load_func=partial(load_audio, sampling_rate=sampling_rate, return_sr=True))
+
+        if inputs.is_multimodal:
+            self._add_default_tags(inputs)
+
+    @staticmethod
+    def _replace_image_tags(inputs: StdTemplateInputs):
+        # compat
+        if inputs.images:
+            return
+        images = []
+        pattern = r'<img>(.+?)</img>'
+        for message in inputs.messages:
+            content = message['content']
+            if not isinstance(content, str):
+                continue
+            for image in re.findall(pattern, content):
+                # only support local_path
+                if os.path.isfile(image):
+                    images.append(image)
+                else:
+                    logger.warning_once(f'Failed to parse image path: `{content}`.', hash_id='<img></img>')
+            message['content'] = re.sub(pattern, '<image>', content)
+        inputs.images = images
+
+    @staticmethod
+    def _replace_start_image_tags(inputs: StdTemplateInputs):
+        # compat
+        generate_mode = False
+        message = inputs.messages[-1]
+        content = message['content']
+        if message['role'] == 'user' and content.endswith('<start-image>'):
+            generate_mode = True
+            message['content'] = message['content'][:-len('<start-image>')]  # remove the <start-image>
+        inputs.generate_mode = generate_mode
+
+    @staticmethod
+    def _extend_tokens(input_ids: List[int], labels: Optional[List[int]], replace_idx_list: List[int],
+                       get_new_tokens: Callable[[int], List[int]]) -> Tuple[List[int], Optional[List[int]]]:
+        added_tokens_len = 0
+        for i, idx in enumerate(replace_idx_list):
+            new_tokens = get_new_tokens(i)
+            token_len = len(new_tokens)
+            input_ids = input_ids[:idx + added_tokens_len] + new_tokens + input_ids[added_tokens_len + idx + 1:]
+            if labels:
+                labels = labels[:idx + added_tokens_len] + [-100] * token_len + labels[added_tokens_len + idx + 1:]
+            added_tokens_len += token_len - 1
+        return input_ids, labels
+
+    def compute_loss_context(self, model, inputs):
+        return nullcontext()
+
+    def _rlhf_encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        chosen_inputs, rejected_inputs = inputs, deepcopy(inputs)
+        assert chosen_inputs.rejected_response is not None, f'inputs: {inputs}'
+        rejected_inputs.messages[-1]['content'] = chosen_inputs.rejected_response
+        chosen_encoded = self._encode_truncated(chosen_inputs)
+        rejected_encoded = self._encode_truncated(rejected_inputs)
+
+        encoded = {}
+        for prefix in ['chosen', 'rejected']:
+            data = locals()[f'{prefix}_encoded']
+            for k, v in data.items():
+                encoded[f'{prefix}_{k}'] = v
+        return encoded
+
+    def _kto_encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        label, inputs.label = inputs.label, None
+        encoded = self._rlhf_encode(inputs)
+        encoded['label'] = bool(label)
+        return encoded
+
+    def _embedding_encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        _encoded = {}
+        labels = []
+
+        def split_multi_medias(_inputs):
+            _content = _inputs.messages[-2]['content']
+            image_size = len(re.findall('<image>', _content))
+            video_size = len(re.findall('<video>', _content))
+            audio_size = len(re.findall('<audio>', _content))
+            _inputs.images = inputs.images[:image_size]
+            assert len(_inputs.images) == image_size
+            inputs.images = inputs.images[image_size:]
+            _inputs.videos = inputs.videos[:video_size]
+            assert len(_inputs.videos) == video_size
+            inputs.videos = inputs.videos[video_size:]
+            _inputs.audios = inputs.audios[:audio_size]
+            assert len(_inputs.audios) == audio_size
+            inputs.audios = inputs.audios[audio_size:]
+
+        anchor = deepcopy(inputs)
+        anchor.messages[-1]['content'] = ''
+        anchor.rejected_response = []
+        split_multi_medias(anchor)
+        anchor_encoded = self._encode_truncated(anchor)
+        for key in anchor_encoded:
+            _encoded[f'anchor_{key}'] = anchor_encoded[key]
+
+        positive = deepcopy(inputs)
+        positive.messages[-2]['content'] = positive.messages[-1]['content']
+        positive.messages[-1]['content'] = ''
+        positive.rejected_response = []
+        split_multi_medias(positive)
+        positive_encoded = self._encode_truncated(positive)
+        for key in positive_encoded:
+            _encoded[f'positive_{key}'] = positive_encoded[key]
+        labels.append(float(inputs.label) if inputs.label is not None else 1.0)
+
+        rejected_len = len(inputs.rejected_response) if inputs.rejected_response else 0
+        for i in range(rejected_len):
+            negative = deepcopy(inputs)
+            negative.messages[-2]['content'] = negative.rejected_response[i]
+            negative.messages[-1]['content'] = ''
+            negative.rejected_response = []
+            split_multi_medias(negative)
+            negative_encoded = self._encode_truncated(negative)
+            for key in negative_encoded:
+                _encoded[f'negative{i}_{key}'] = negative_encoded[key]
+            labels.append(0.0)
+
+        _encoded['labels'] = labels
+        return _encoded
+
+    def _seq_cls_encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = self._encode_truncated(inputs)
+        encoded.pop('labels', None)
+        if inputs.label is not None:
+            labels = inputs.label
+            problem_type = self._get_problem_type(self.config, labels=labels)
+            if problem_type == 'single_label_classification':
+                labels = int(labels)
+            encoded['labels'] = labels
+        return encoded
+
+    @torch.inference_mode()
+    def encode(self,
+               inputs: Union[TemplateInputs, Dict[str, Any], InferRequest],
+               return_template_inputs: bool = False) -> Dict[str, Any]:
+        """The entrance method of Template!
+
+        Returns:
+            return {'input_ids': List[int], 'labels': Optional[List[int]], ...}
+        """
+        if isinstance(inputs, (InferRequest, TemplateInputs)):
+            inputs = asdict(inputs)
+
+        if isinstance(inputs, dict):
+            inputs = deepcopy(inputs)
+            if not self.is_training:
+                InferRequest.remove_response(inputs['messages'])
+            inputs = StdTemplateInputs.from_dict(inputs)
+        elif isinstance(inputs, StdTemplateInputs):
+            inputs = deepcopy(inputs)
+        assert isinstance(inputs, StdTemplateInputs)
+        self._preprocess_inputs(inputs)
+
+        if self.mode in {'pt', 'train', 'prm', 'vllm', 'lmdeploy'}:
+            encoded = self._encode_truncated(inputs)
+        elif self.mode == 'seq_cls':
+            encoded = self._seq_cls_encode(inputs)
+        elif self.mode == 'rlhf':
+            encoded = self._rlhf_encode(inputs)
+        elif self.mode == 'kto':
+            encoded = self._kto_encode(inputs)
+        elif self.mode == 'embedding':
+            encoded = self._embedding_encode(inputs)
+        for key in list(encoded.keys()):
+            if encoded[key] is None:
+                encoded.pop(key)
+        if return_template_inputs:
+            encoded['template_inputs'] = inputs
+        return encoded
+
+    def packing_row(self, row: List[Tuple[Dict[str, Any], int]]) -> Dict[str, Any]:
+        packed = {}
+        keys = set()
+        for r in row:
+            keys.update(r[0].keys())
+        for key in keys:
+            if key in {'input_ids', 'labels', 'loss_scale'}:
+                packed[key] = sum((x[0][key] for x in row), start=[])
+        if 'position_ids' not in packed:
+            packed['position_ids'] = sum((list(range(x[1])) for x in row), start=[])
+
+        packed.update(self._data_collator_mm_data([r[0] for r in row]))
+        return packed
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        return inputs
+
+    @staticmethod
+    def _skip_stop_tokens(generate_ids: List[int], stop_tokens: List[int], is_finished: bool) -> List[int]:
+        len_tokens = len(stop_tokens)
+        if is_finished and generate_ids[-len_tokens:] == stop_tokens:
+            return generate_ids[:-len_tokens]
+        if not is_finished:
+            for i in range(len_tokens, 0, -1):
+                if generate_ids[-i:] == stop_tokens[:i]:
+                    return generate_ids[:-i]
+        return generate_ids
+
+    @staticmethod
+    def _get_seq_cls_logprobs(pred: int, logprobs: torch.Tensor, top_logprobs: int):
+        idxs = logprobs.argsort(descending=True, dim=-1)[:top_logprobs].tolist()
+        logprobs = logprobs.tolist()
+        return {
+            'content': [{
+                'index': pred,
+                'logprobs': [logprobs[p] for p in pred] if isinstance(pred, (list, tuple)) else logprobs[pred],
+                'top_logprobs': [{
+                    'index': idx,
+                    'logprob': logprobs[idx]
+                } for idx in idxs]
+            }]
+        }
+
+    @staticmethod
+    def _get_problem_type(config, labels=None, logits=None) -> str:
+        problem_type = config.problem_type
+        if problem_type is not None:
+            return problem_type
+        if labels is not None:
+            if isinstance(labels, (list, tuple)):
+                if labels and isinstance(labels[0], float):
+                    problem_type = 'regression'
+                else:
+                    problem_type = 'multi_label_classification'
+            else:
+                problem_type = 'single_label_classification'
+                assert config.num_labels >= labels + 1
+        if logits is not None:
+            if logits.shape[-1] == 1:
+                problem_type = 'regression'
+            else:
+                problem_type = 'single_label_classification'  # compatible with older versions
+        assert problem_type is not None
+        config.problem_type = problem_type
+        return problem_type
+
+    def decode_seq_cls(self, logits: torch.Tensor, top_logprobs: int):
+        assert isinstance(logits, torch.Tensor)
+        problem_type = self._get_problem_type(self.config, logits=logits)
+        if problem_type == 'regression':
+            preds = logits.squeeze(dim=-1).tolist()
+            logprobs = [None] * len(preds)
+        else:
+            if problem_type == 'single_label_classification':
+                preds = torch.argmax(logits, dim=-1).tolist()
+                logprobs = torch.log_softmax(logits, -1)
+            else:
+                preds = [(logprob >= 0.5).nonzero(as_tuple=True)[0].tolist() for logprob in torch.sigmoid(logits)]
+                logprobs = F.logsigmoid(logits)
+            logprobs = [self._get_seq_cls_logprobs(pred, logprobs[i], top_logprobs) for i, pred in enumerate(preds)]
+        return preds, logprobs
+
+    def decode(self,
+               generate_ids: List[int],
+               *,
+               is_finished: bool = True,
+               tokenizer_kwargs=None,
+               first_token=True,
+               **kwargs) -> Any:
+        tokenizer_kwargs = tokenizer_kwargs or {}
+        response = self._skip_stop_decode(generate_ids, is_finished, **tokenizer_kwargs)
+        if first_token and self.template_meta.response_prefix:
+            response = self.template_meta.response_prefix + response
+        return response
+
+    def decode_prm(self, input_ids: torch.Tensor, logits: torch.Tensor) -> Any:
+        raise NotImplementedError
+
+    def generate(self, model, *args, **kwargs):
+        if isinstance(model, PeftModel):
+            signature = inspect.signature(model.model.generate)
+        else:
+            signature = inspect.signature(model.generate)
+        if 'use_model_defaults' in signature.parameters and 'use_model_defaults' not in kwargs:
+            kwargs['use_model_defaults'] = False
+        return model.generate(*args, **kwargs)
+
+    def _skip_stop_decode(self, generate_ids: List[int], is_finished: bool, **decode_kwargs) -> Any:
+        # Do not print template_meta.suffix[-1] and eos_token.
+        # However, other stop_words will be printed.
+        tokenizer = self.tokenizer
+
+        if len(generate_ids) > 0 and generate_ids[-1] == tokenizer.eos_token_id:
+            generate_ids = generate_ids[:-1]
+        # skip suffix and eos_token
+        template_suffix = self.template_meta.suffix[-1]
+        if isinstance(template_suffix, str):
+            # [-1:]: fix OpenGVLab/Mini-InternVL-Chat-4B-V1-5
+            template_suffix = tokenizer.encode(template_suffix, add_special_tokens=False)[-1:]
+        generate_ids = self._skip_stop_tokens(generate_ids, template_suffix, is_finished)
+        if 'spaces_between_special_tokens' not in decode_kwargs:
+            decode_kwargs['spaces_between_special_tokens'] = False
+        return tokenizer.decode(generate_ids, **decode_kwargs)
+
+    def prepare_generate_kwargs(self, generate_kwargs: Dict[str, Any], *, model=None) -> Dict[str, Any]:
+        generation_config = generate_kwargs['generation_config']
+        stop_words = getattr(generation_config, 'stop_words', None) or self.template_meta.stop_words
+        generate_kwargs['stopping_criteria'] = StoppingCriteriaList([StopWordsCriteria(self.tokenizer, stop_words)])
+        return generate_kwargs
+
+    @staticmethod
+    def _save_pil_image(image: Image.Image) -> str:
+        img_bytes = image.tobytes()
+        img_hash = hashlib.sha256(img_bytes).hexdigest()
+        tmp_dir = os.path.join(get_cache_dir(), 'tmp', 'images')
+        logger.info_once(f'create tmp_dir: {tmp_dir}')
+        os.makedirs(tmp_dir, exist_ok=True)
+        img_path = os.path.join(tmp_dir, f'{img_hash}.png')
+        if not os.path.exists(img_path):
+            image.save(img_path)
+        return img_path
+
+    @staticmethod
+    def _concat_context_list(
+            context_list: List[Context],
+            res_context_list: List[Context],  # inplace
+            res_context_type: List[ContextType],  # inplace
+            system: Optional[str] = None,
+            query: Optional[str] = None,
+            response: Optional[str] = None,
+            round0: Optional[int] = None) -> None:
+        """Concat context list and replace placeholder"""
+        round1 = None
+        if round0 is not None:
+            round1 = str(round0 + 1)
+            round0 = str(round0)
+        for context in context_list:
+            if isinstance(context, str):
+                if '{{RESPONSE}}' == context:
+                    assert response is not None
+                    res_context_list.append(response)
+                    res_context_type.append(ContextType.RESPONSE)
+                    continue
+                old_str_list = ['{{SYSTEM}}', '{{QUERY}}', '{{ROUND0}}', '{{ROUND1}}']
+                new_str_list = [system, query, round0, round1]
+                for (old_str, new_str) in zip(old_str_list, new_str_list):
+                    if new_str is not None and old_str in context:
+                        assert isinstance(new_str, str), f'new_str: {new_str}'
+                        context = context.replace(old_str, new_str)
+            if len(context) == 0:
+                continue
+            res_context_list.append(context)
+            res_context_type.append(ContextType.OTHER)
+
+    def _simplify_context_list(self, context_list: List[Context], loss_scale_list: List[float],
+                               inputs: StdTemplateInputs) -> Tuple[List[Context], List[float]]:
+        """Merge anything in the context to simplify the inputs"""
+        context_list, loss_scale_list = self._split_special_tokens(context_list, loss_scale_list)
+        context_list, loss_scale_list = self._pre_tokenize(context_list, loss_scale_list, inputs)
+
+        res: List[Context] = []  # result of context_list
+        res_loss_scale: List[float] = []  # result of loss_scale_list
+        temp: List[str] = []
+        temp_loss_scale = 0.
+        for i, (context, loss_scale) in enumerate(zip(context_list, loss_scale_list)):
+            if isinstance(context, str) and (loss_scale == temp_loss_scale):
+                temp.append(context)
+            else:
+                if len(temp) > 0:
+                    res.append(''.join(temp))
+                    res_loss_scale.append(temp_loss_scale)
+                    temp.clear()
+                if isinstance(context, str):  # loss_scale diff
+                    temp.append(context)
+                else:
+                    res.append(context)
+                    res_loss_scale.append(loss_scale)
+                temp_loss_scale = loss_scale
+        if len(temp) > 0:
+            res.append(''.join(temp))
+            res_loss_scale.append(temp_loss_scale)
+
+        return res, res_loss_scale
+
+    @staticmethod
+    def _split_special_tokens(context_list: List[Context],
+                              loss_scale_list: List[float]) -> Tuple[List[Context], List[float]]:
+        """Split special tokens, for example `<image>`, `<video>`, this will help the replace_tag operation"""
+        res: List[Context] = []
+        loss_scale_res: List[float] = []
+        for context, loss_scale in zip(context_list, loss_scale_list):
+            contexts = []
+            if isinstance(fetch_one(context), str):
+                for d in split_str_parts_by(context, Template.special_tokens):
+                    contexts.extend([d['key'], d['content']])
+                contexts = [c for c in contexts if c]
+                res.extend(contexts)
+                loss_scale_res.extend([loss_scale] * len(contexts))
+            else:
+                res.append(context)
+                loss_scale_res.append(loss_scale)
+        return res, loss_scale_res
+
+    def _tokenize(self, context, **tokenizer_kwargs):
+        return self.tokenizer(
+            context, return_attention_mask=False, add_special_tokens=False, **tokenizer_kwargs)['input_ids']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        """Override this function to do your own replace operation.
+
+        This method is used to replace standard tags like `<image>` to some tokens that the model needs.
+
+        Args:
+            media_type: The modal.
+            index: The index of the medias, for index 0 represents the first elements in `images`
+            inputs: The inputs
+
+        Returns:
+            The content or input_ids after replacement.
+        """
+        if media_type == 'image':
+            if self.mode == 'lmdeploy':
+                return [[-100]]
+            return self.image_placeholder
+        elif media_type == 'video':
+            return self.video_placeholder
+        elif media_type == 'audio':
+            return self.audio_placeholder
+
+    def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:
+        """Replace objects referenced by the bbox to contents or input_ids. This is useful in the grounding task.
+        Override this function to do your own replace operation.
+
+        Args:
+            ref: Description of the bbox
+            index: The index in the `objects` key
+            inputs: The inputs
+
+        Returns:
+            The contents or input_ids replaced
+        """
+        return [ref]
+
+    def replace_cot_process(self, inputs: StdTemplateInputs) -> List[Context]:
+        """Replace the cot process label for PRM training or inference.
+        Override this function to do your own replace operation.
+
+        Args:
+            inputs: The inputs
+
+        Returns:
+            The contents or input_ids replaced
+        """
+        return [self.cot_process_placeholder]
+
+    @staticmethod
+    def _get_bbox_str(bbox: List[int]) -> str:
+        point = []
+        for x, y in zip(bbox[::2], bbox[1::2]):
+            point.append(f'({x},{y})')
+        return ','.join(point)
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        """Replace bbox pointing to the objects to contents or input_ids. This is useful in the grounding task.
+        Override this function to do your own replace operation.
+
+        Args:
+            bbox: [x, y] or [x1, y1, x2, y2]
+            index: The index in the `objects` key
+            inputs: The inputs
+
+        Returns:
+            The contents or input_ids replaced
+        """
+        return [f'[{self._get_bbox_str(bbox)}]']
+
+    def _pre_tokenize_images(self, context_list: List[Context], loss_scale_list: List[float],
+                             inputs: StdTemplateInputs) -> Tuple[List[Context], List[float]]:
+        # https://github.com/modelscope/ms-swift/issues/3407
+        # Fix the bounding box position offset issue in the Qwen2.5-VL grounding task.
+        res: List[Context] = []
+        res_loss_scale: List[float] = []
+        inputs.image_idx = 0
+
+        for context, loss_scale in zip(context_list, loss_scale_list):
+            if context == '<image>' and inputs.is_multimodal and inputs.image_idx < len(inputs.images):
+                c_list = self.replace_tag('image', inputs.image_idx, inputs)
+                inputs.image_idx += 1
+                loss_scale = 0. if self.template_backend == 'swift' else 1.
+            else:
+                c_list = [context]
+            res += c_list
+            res_loss_scale += [loss_scale] * len(c_list)
+        return res, res_loss_scale
+
+    def _pre_tokenize(self, context_list: List[Context], loss_scale_list: List[float],
+                      inputs: StdTemplateInputs) -> Tuple[List[Context], List[float]]:
+        """This method happens before tokenization, replace standard tags to the contents or input_ids needed by
+        the model.
+
+        Args:
+            context_list: The content list
+            loss_scale_list: The loss scale list
+        Returns:
+            The context_list and loss_scale_list after replacement.
+        """
+        context_list, loss_scale_list = self._pre_tokenize_images(context_list, loss_scale_list, inputs)
+        if inputs.images and inputs.objects:
+            self.normalize_bbox(inputs)
+        # replace tag/object/box
+        res: List[Context] = []  # result of context_list
+        res_loss_scale: List[float] = []  # result of loss_scale_list
+
+        # reset
+        for k in ['video', 'audio', 'object', 'box']:
+            setattr(inputs, f'{k}_idx', 0)
+
+        for context, loss_scale in zip(context_list, loss_scale_list):
+            for k in ['video', 'audio']:
+                if context == f'<{k}>' and inputs.is_multimodal and getattr(inputs, f'{k}_idx') < len(
+                        getattr(inputs, f'{k}s')):
+                    c_list = self.replace_tag(k, getattr(inputs, f'{k}_idx'), inputs)
+                    setattr(inputs, f'{k}_idx', getattr(inputs, f'{k}_idx') + 1)
+                    loss_scale = 0.
+                    break
+            else:
+                ref = inputs.objects.get('ref') or []
+                bbox = inputs.objects.get('bbox') or []
+                if context == '<ref-object>' and inputs.ref_idx < len(ref):
+                    idx = inputs.ref_idx
+                    c_list = self.replace_ref(ref[idx], idx, inputs)
+                    inputs.ref_idx += 1
+                elif context == '<bbox>' and inputs.bbox_idx < len(bbox):
+                    idx = inputs.bbox_idx
+                    c_list = self.replace_bbox(bbox[idx], idx, inputs)
+                    inputs.bbox_idx += 1
+                elif context == '<cot-process>' and self.mode == 'prm':
+                    c_list = self.replace_cot_process(inputs)
+                else:
+                    c_list = [context]
+            res += c_list
+            res_loss_scale += [loss_scale] * len(c_list)
+        return res, res_loss_scale
+
+    @staticmethod
+    def _add_default_tags(inputs: StdTemplateInputs):
+        total_content = '\n'.join([message['content'] or '' for message in inputs.messages])
+        if inputs.rejected_response:
+            if isinstance(inputs.rejected_response, str):
+                total_content += inputs.rejected_response
+            else:
+                total_content += '\n'.join(inputs.rejected_response)
+        if inputs.system:
+            total_content = f'{inputs.system}\n{total_content}'
+        for media_type in ['image', 'audio', 'video']:
+            media_key, media_tag = f'{media_type}s', f'<{media_type}>'
+            medias = getattr(inputs, media_key)
+            if not isinstance(medias, list):
+                medias = [medias]
+            if medias:
+                num_media_tags = len(re.findall(media_tag, total_content))
+                num_media = len(medias)
+                num_new_tags = num_media - num_media_tags
+                if num_new_tags > 0:
+                    inputs.messages[0]['content'] = media_tag * num_new_tags + inputs.messages[0]['content']
+                elif num_new_tags < 0:
+                    logger.warning(
+                        f'num_media: {num_media}, num_media_tags: {num_media_tags}, total_content: {total_content}. '
+                        'We will only replace the frontmost media_tags while keeping the subsequent media_tags.')
+
+    def _encode_context_list(
+            self,
+            context_list: List[Context],
+            loss_scale_list: Optional[List[float]] = None) -> Tuple[List[int], List[int], List[float], Dict[str, Any]]:
+        """return: input_ids, labels, tokenizer_kwargs"""
+        input_ids: List[int] = []
+        labels: List[int] = []
+        loss_scale: List[float] = []
+        tokenizer_kwargs = {}
+        if loss_scale_list is None:
+            loss_scale_list = [0.] * len(context_list)
+        ignore_loss_scale = all(loss_scale in {0, 1} for loss_scale in loss_scale_list)
+        for i, (context, loss_weight) in enumerate(zip(context_list, loss_scale_list)):
+            if isinstance(context, str):
+                # tokenizer_kwargs is the returned tokenizer_kwargs,
+                # while curr_tokenizer_kwargs is the tokenizer_kwargs for the current context.
+                token_list = self._tokenize(context)
+            else:
+                token_list = context
+            input_ids += token_list
+            if loss_scale_list[i] > 0.0:
+                labels += token_list
+            else:
+                labels += [-100] * len(token_list)
+            if not ignore_loss_scale:
+                loss_scale.extend([loss_weight] * len(token_list))
+        if ignore_loss_scale:
+            loss_scale = None
+        return input_ids, labels, loss_scale, tokenizer_kwargs
+
+    @staticmethod
+    def _add_dynamic_eos(input_ids: List[int], labels: List[int], loss_scale: Optional[List[int]],
+                         suffix_tokens_id: List[int]) -> None:
+        suffix_len = len(suffix_tokens_id)
+        start = 0
+        for i in range(1, len(labels)):
+            if labels[i - 1] >= 0 and labels[i] == -100:
+                start = i
+            if start > 0 and labels[i - 1] == -100 and labels[i] >= 0:
+                # [0, 1, 2, -100(start), -100, 3(i), 4]
+                length = i - start
+                if length >= suffix_len and input_ids[start:start + suffix_len] == suffix_tokens_id:
+                    labels[start:start + suffix_len] = suffix_tokens_id
+                    if loss_scale and loss_scale[start:start + suffix_len] == [0] * suffix_len:
+                        loss_scale[start:start + suffix_len] = [1] * suffix_len
+
+    @staticmethod
+    def _get_std_messages(messages):
+        if messages and messages[0]['role'] == 'assistant':
+            messages.insert(0, {'role': 'user', 'content': ''})  # pretrain
+        if len(messages) % 2 == 1:
+            messages.append({'role': 'assistant', 'content': None})  # inference
+
+    def _jinja_encode(self, inputs: StdTemplateInputs):
+        messages = inputs.messages.copy()
+        if inputs.system is not None:
+            messages.insert(0, {'role': 'system', 'content': inputs.system})
+        if messages[-1]['content'] is None:
+            messages.pop()
+        add_generation_prompt = messages[-1]['role'] != 'assistant'
+        kwargs = {}
+        if inputs.tools:
+            kwargs['tools'] = inputs.tools
+        text = self.tokenizer.apply_chat_template(
+            messages, tokenize=False, add_generation_prompt=add_generation_prompt, **kwargs)
+        answer_len = 1 if self.is_training else 0
+        return [text], [1.], answer_len
+
+    def _get_system(self, inputs) -> Optional[str]:
+        template_meta = self.template_meta
+        system = inputs.system
+        tools = inputs.tools
+        template_meta.check_system(system)
+        if system is None:
+            system = template_meta.default_system
+
+        if tools is not None:
+            system = self.agent_template._format_tools(tools, system or '', inputs.messages[0])
+        return system
+
+    @staticmethod
+    def _swift_prepare_function_call(agent_template, messages):
+        if len(messages) < 2:
+            return
+        i = 1
+        while i < len(messages):
+            pre_message, message = messages[i - 1], messages[i]
+            pre_role, pre_content = pre_message['role'], pre_message['content']
+            role, content = message['role'], message['content']
+            if pre_role == 'assistant' and role == 'tool':
+                i_start = i
+                while i + 1 < len(messages) and messages[i + 1]['role'] == 'tool':
+                    i += 1
+                pre_message['content'], tool_content = agent_template._format_tool_responses(
+                    pre_content, messages[i_start:i + 1])
+                messages[i_start:i + 1] = [{'role': 'tool', 'content': tool_content}]
+                i = i_start + 1
+            elif pre_role == 'assistant' and role == 'assistant':
+                # Consecutive messages from the assistant role need to be merged to prevent errors.
+                pre_message['content'] = pre_content + content
+                messages.pop(i)
+            else:
+                i += 1
+
+    def _swift_encode(self, inputs: StdTemplateInputs):
+        template_meta = self.template_meta
+        system = self._get_system(inputs)
+        self._swift_prepare_function_call(self.agent_template, inputs.messages)
+
+        self._get_std_messages(inputs.messages)
+        n_round = len(inputs.messages) // 2
+        if n_round > 1 and not self.template_meta.support_multi_round:
+            logger.warning_once(
+                'The template does not support multi-round chat. Only use the last round of the conversation.')
+            inputs.messages = inputs.messages[-2:]
+
+        res_context_list: List[Context] = []
+        res_context_types: List[ContextType] = []
+        sep_token = None
+        if template_meta.auto_add_bos:
+            all_tokens = self.tokenizer.encode('a')
+            single_token = self.tokenizer.encode('a', add_special_tokens=False)
+            assert len(single_token) == 1
+            idx = all_tokens.index(single_token[0])
+            bos_token = all_tokens[:idx]
+            sep_token = all_tokens[idx + 1:]
+            if bos_token:
+                res_context_list.append(bos_token)
+                res_context_types.append(ContextType.OTHER)
+
+        prefix = template_meta.system_prefix if system else template_meta.prefix
+        self._concat_context_list(prefix, res_context_list, res_context_types, system=system)
+
+        n_round = len(inputs.messages) // 2
+        for i, (query_message, response_message) in enumerate(zip(inputs.messages[::2], inputs.messages[1::2])):
+            query_role, query = query_message['role'], query_message['content']
+            response_role, response = response_message['role'], response_message['content']
+            # TODO: Optimize the Template mechanism.
+            assert query_role in {'user', 'tool'}, f'query_role: {query_role}'
+            assert response_role in {'assistant'}, f'response_role: {response_role}'
+            if query_role == 'tool':
+                prompt = query
+                query = ''
+            elif template_meta.is_post_system and i == n_round - 1:
+                prompt = template_meta.system_prompt
+            else:
+                prompt = template_meta.prompt
+
+            context_list = prompt.copy()
+            extra_context_list = []
+            extra_context_type = None
+            if i < n_round - 1:
+                # Not the last round.
+                context_list.append('{{RESPONSE}}')
+                if inputs.messages[2 * (i + 1)]['role'] != 'tool':
+                    extra_context_list = template_meta.chat_sep
+                    extra_context_type = ContextType.OTHER
+            elif response is not None:
+                # It is the final round, and the response exists (during training).
+                context_list.append('{{RESPONSE}}')
+                if self.is_training and not sep_token:
+                    extra_context_list = template_meta.suffix
+                    extra_context_type = ContextType.SUFFIX
+            elif template_meta.response_prefix:
+                # final round and during inference.
+                context_list.append(template_meta.response_prefix)
+
+            self._concat_context_list(
+                context_list,
+                res_context_list,
+                res_context_types,
+                query=query,
+                response=response,
+                system=system,
+                round0=i)
+            res_context_list += extra_context_list
+            res_context_types += [extra_context_type] * len(extra_context_list)
+        if template_meta.auto_add_bos and sep_token:
+            res_context_list.append(sep_token)
+            res_context_types.append(ContextType.SUFFIX)
+        from swift.plugin import loss_scale_map
+        res_context_list, loss_scale_list = loss_scale_map[self.loss_scale](res_context_list, res_context_types,
+                                                                            inputs.messages)
+        if self.is_training:
+            answer_len = len(extra_context_list) + bool(response is not None)
+        else:
+            answer_len = 0
+        return res_context_list, loss_scale_list, answer_len
+
+    def _encode_truncated(self, inputs):
+        if self.mode in {'vllm', 'lmdeploy'}:
+            encoded = Template._encode(self, inputs)
+            for key in ['images', 'audios', 'videos']:
+                encoded[key] = getattr(inputs, key)
+        else:
+            encoded = self._encode(inputs)
+
+        input_ids = encoded.get('input_ids')
+        labels = encoded.get('labels')
+        loss_scale = encoded.get('loss_scale')
+        if self.max_length is not None:
+            if self.truncation_strategy == 'right':
+                input_ids = input_ids[:self.max_length]
+                if labels is not None:
+                    labels = labels[:self.max_length]
+                if loss_scale is not None:
+                    loss_scale = loss_scale[:self.max_length]
+            elif self.truncation_strategy == 'left':
+                if len(input_ids) > self.max_length:
+                    logger.warning_once(
+                        'Input data was left-truncated because its length exceeds `max_length` (input length: '
+                        f'{len(input_ids)}, max_length: {self.max_length}). '
+                        'This may cause loss of important tokens (e.g., image tokens) and lead to errors. '
+                        'To avoid this, consider increasing `max_length` or pre-filtering long sequences.',
+                        hash_id='max_length_check')
+                input_ids = input_ids[-self.max_length:]
+                if labels is not None:
+                    labels = labels[-self.max_length:]
+                if loss_scale is not None:
+                    loss_scale = loss_scale[-self.max_length:]
+            elif self.truncation_strategy == 'raise':
+                length = len(input_ids or labels or [])
+                if length > self.max_length:
+                    raise MaxLengthError(f'Current length of row({length}) is larger'
+                                         f' than the max_length({self.max_length}).')
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        template_backend = self.template_backend
+        if (self.template_meta.template_type == 'dummy' and self.use_chat_template and not self.is_training
+                and self.mode != 'seq_cls'):
+            template_backend = 'jinja'
+            logger.info_once(f'Setting template_backend: {template_backend}')
+        res_context_list, loss_scale_list, answer_len = (
+            self._swift_encode(inputs) if template_backend == 'swift' else self._jinja_encode(inputs))
+        encoded = {}
+        if self.is_encoder_decoder:
+            # tokenizer_kwargs: use prompt (qwen-audio)
+            total_len = len(res_context_list)
+            for key, _slice in zip(['prompt', 'answer'],
+                                   [slice(0, total_len - answer_len),
+                                    slice(total_len - answer_len, total_len)]):
+                context_list, loss_scale = self._simplify_context_list(res_context_list[_slice],
+                                                                       loss_scale_list[_slice], inputs)
+                input_ids, labels, loss_scale, tokenizer_kwargs = self._encode_context_list(context_list, loss_scale)
+                encoded[f'{key}_input_ids'] = input_ids
+                if key == 'answer':
+                    encoded['labels'] = labels
+                    encoded['loss_scale'] = loss_scale
+            input_ids = encoded['prompt_input_ids'] + encoded['answer_input_ids']
+        else:
+            res_context_list, loss_scale_list = self._simplify_context_list(res_context_list, loss_scale_list, inputs)
+            input_ids, labels, loss_scale, tokenizer_kwargs = self._encode_context_list(
+                res_context_list, loss_scale_list)
+        self._add_dynamic_eos(input_ids, labels, loss_scale, self._encode_context_list(self.template_meta.suffix)[0])
+
+        if tokenizer_kwargs:
+            encoded['tokenizer_kwargs'] = tokenizer_kwargs
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        if self.use_megatron:
+            self._handle_megatron_cp(encoded)
+            encoded['labels'] = encoded['labels'][1:] + [-100]
+            encoded['position_ids'] = list(range(len(encoded['labels'])))
+        elif encoded.get('labels') is not None:
+            encoded['labels'][0] = -100
+        if not self.is_training:
+            for k in list(encoded.keys()):
+                if k.endswith('labels') or k.endswith('loss_scale'):
+                    encoded[k] = None
+        return encoded
+
+    def _handle_megatron_cp(self, encoded: Dict[str, Any]) -> None:
+        cp_size = self.sequence_parallel_size
+        if cp_size == 1:
+            return
+        input_ids = encoded['input_ids']
+        padding_len = math.ceil(len(input_ids) / (cp_size * 2)) * (cp_size * 2) - len(input_ids)
+        input_ids += [self.tokenizer.pad_token_id] * padding_len
+        encoded['labels'] += [-100] * padding_len
+
+    def debug_logger(self, inputs):
+        if not strtobool(os.getenv('SWIFT_DEBUG', 'false')):
+            return
+        if 'input_ids' in inputs:
+            k = 'input_ids'
+            val = inputs['input_ids']
+        elif 'generate_ids' in inputs:
+            k = 'generate_ids'
+            val = inputs['generate_ids']
+        for v in val:
+            self.print_inputs({k: v.tolist()})
+
+    @staticmethod
+    def _split_list(inputs: List[int], x: int) -> List[List[int]]:
+        idxs = findall(inputs, x)
+        idxs.append(len(inputs))
+        res = []
+        lo = 0
+        for idx in idxs:
+            res.append(inputs[lo:idx])
+            lo = idx + 1
+        return res
+
+    def replace_video2image(self, load_video_func, inputs, replace_tag: Callable) -> List[Context]:
+        context_list = []
+        if self.mode in {'vllm', 'lmdeploy'}:
+            video = inputs.videos.pop(inputs.video_idx)
+            inputs.video_idx -= 1
+        else:
+            video = inputs.videos[inputs.video_idx]
+        images = inputs.images
+        new_images = load_video_func(video)
+        inputs.images = images[:inputs.image_idx] + new_images + images[inputs.image_idx:]
+        for i in range(len(new_images)):
+            context_list += replace_tag(i)
+        inputs.image_idx += len(new_images)
+        return context_list
+
+    def get_generate_ids(self, generate_ids: Union[torch.Tensor, List[int]],
+                         num_prompt_tokens: int) -> Union[torch.Tensor, List[int]]:
+        if self.skip_prompt:
+            generate_ids = generate_ids[..., num_prompt_tokens:]
+        return generate_ids
+
+    def post_process_generate_response(self, response: str, inputs: StdTemplateInputs) -> str:
+        return response
+
+    def pre_forward_hook(self, model: nn.Module, args, kwargs):
+        from swift.llm import to_device
+        old_kwargs = to_device(kwargs, model.device)
+        kwargs = to_device(self._post_encode(model, old_kwargs), model.device)
+        for k, v in old_kwargs.items():
+            if k in {'input_ids', 'attention_mask', 'labels', 'position_ids'} and k not in kwargs:
+                kwargs[k] = v
+        if 'inputs_embeds' in kwargs:
+            kwargs.pop('input_ids', None)
+
+        if isinstance(model, PeftModel):
+            parameters = inspect.signature(model.model.forward).parameters
+        else:
+            parameters = inspect.signature(model.forward).parameters
+        if 'position_ids' not in parameters:
+            kwargs.pop('position_ids', None)
+        return args, kwargs
+
+    @property
+    def is_training(self):
+        return self.mode not in {'vllm', 'lmdeploy', 'pt'}
+
+    def set_mode(self, mode: Literal['vllm', 'lmdeploy', 'pt', 'seq_cls', 'train', 'rlhf', 'kto']) -> None:
+        self.mode = mode
+
+    def register_post_encode_hook(self, models: List[nn.Module]) -> None:
+        """This function is important for multi-modal training, as it registers the post_encode method
+            as a forward hook, converting input_ids into inputs_embeds.
+        """
+        if self._handles:
+            return
+
+        for model in models:
+            # please use torch>=2.0
+            handle = model.register_forward_pre_hook(self.pre_forward_hook, with_kwargs=True)
+            self._handles.append((model, handle))
+
+        if is_deepspeed_zero3_enabled():
+            import deepspeed
+            self._deepspeed_initialize = deepspeed.initialize
+
+            @wraps(self._deepspeed_initialize)
+            def _initialize(*args, **kwargs):
+                res = self._deepspeed_initialize(*args, **kwargs)
+                for model, handle in self._handles:
+                    model._forward_pre_hooks.move_to_end(handle.id)
+                return res
+
+            deepspeed.initialize = _initialize
+
+    def remove_post_encode_hook(self):
+        models = []
+        for model, handle in self._handles:
+            models.append(model)
+            handle.remove()
+        self._handles = []
+
+        if self._deepspeed_initialize is not None:
+            import deepspeed
+            deepspeed.initialize = self._deepspeed_initialize
+        self._deepspeed_initialize = None
+        return models
+
+    def data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        if self.mode == 'rlhf':
+            return self._rlhf_data_collator(batch, padding_to=padding_to)
+        elif self.mode == 'kto':
+            return self._kto_data_collator(batch, padding_to=padding_to)
+        elif self.mode in {'pt', 'train', 'prm'}:
+            return self._data_collator(batch, padding_to=padding_to)
+        elif self.mode == 'seq_cls':
+            return self._seq_cls_data_collator(batch, padding_to=padding_to)
+        elif self.mode == 'embedding':
+            return self._embedding_data_collator(batch, padding_to=padding_to)
+
+    @staticmethod
+    def _fetch_inputs_startswith(batch: List[Dict[str, Any]], prefix: str) -> List[Dict[str, Any]]:
+        new_batch = []
+        for inputs in batch:
+            new_inputs = {}
+            for k, v in inputs.items():
+                if k.startswith(prefix):
+                    new_inputs[k[len(prefix):]] = v
+            new_batch.append(new_inputs)
+        return new_batch
+
+    @staticmethod
+    def fetch_inputs(batch: List[Dict[str, Any]], keys: Optional[List[str]] = None) -> Dict[str, Any]:
+        from swift.llm import RowPreprocessor
+        keys = keys or []
+        rows = RowPreprocessor.rows_to_batched(batch)
+        return {k: rows[k] for k in keys if rows.get(k) is not None}
+
+    @staticmethod
+    def gather_list(batch: List[Dict[str, Any]], attr_name: str) -> Optional[List[Any]]:
+        # List[Tensor] ->  List[Tensor]
+        res = []
+        for b in batch:
+            if b.get(attr_name) is not None:
+                res += b.pop(attr_name)
+        return res
+
+    @staticmethod
+    def concat_tensor(batch: List[Dict[str, Any]], attr_name: str, dim: int) -> Optional[torch.Tensor]:
+        res = []
+        for b in batch:
+            if b.get(attr_name) is not None:
+                res.append(b.pop(attr_name))
+        return torch.concat(res, dim=dim) if res else None
+
+    def _rlhf_data_collator(self,
+                            batch: List[Dict[str, Any]],
+                            *,
+                            chosen_prefix: str = 'chosen_',
+                            rejected_prefix: str = 'rejected_',
+                            padding_to: Optional[int] = None) -> Dict[str, Any]:
+        new_batch = []
+        for prefix in [chosen_prefix, rejected_prefix]:
+            new_batch += self._fetch_inputs_startswith(batch, prefix)
+        return self._data_collator(new_batch, padding_to=padding_to)
+
+    def _kto_data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        new_batch = self._fetch_inputs_startswith(batch, 'chosen_')
+        kl_batch = self._fetch_inputs_startswith(batch, 'rejected_')
+
+        res = self._data_collator(new_batch, padding_to=padding_to)
+        kl_res = self._data_collator(kl_batch, padding_to=padding_to)
+        res = {
+            **{f'completion_{k}': v
+               for k, v in res.items()},
+            **{f'KL_completion_{k}': v
+               for k, v in kl_res.items()},
+        }
+        label = [b['label'] for b in batch if b.get('label') is not None]
+        if label:
+            res['label'] = label
+        return res
+
+    def _embedding_data_collator(self,
+                                 batch: List[Dict[str, Any]],
+                                 *,
+                                 padding_to: Optional[int] = None) -> Dict[str, Any]:
+        labels = []
+        new_batch = []
+        for b in batch:
+            keys = [key for key in b.keys() if 'negative' in key]
+            max_neg = max([int(re.findall(r'negative(-?\d+)', key)[0]) for key in keys]) if keys else None
+            indexes = ['anchor_', 'positive_']
+            if max_neg is not None:
+                for i in range(0, max_neg + 1):
+                    indexes.append(f'negative{i}_')
+            for prefix in indexes:
+                new_batch += self._fetch_inputs_startswith([b], prefix)
+            labels.extend(b.get('labels', None))
+        res = self._data_collator(new_batch, padding_to=padding_to)
+        if labels:
+            res['labels'] = torch.tensor(labels, dtype=torch.float32)
+        return res
+
+    def _seq_cls_data_collator(self,
+                               batch: List[Dict[str, Any]],
+                               *,
+                               padding_to: Optional[int] = None) -> Dict[str, Any]:
+        labels = [b.pop('labels') for b in batch if b.get('labels') is not None]
+        res = self._data_collator(batch, padding_to=padding_to)
+        if labels:
+            problem_type = self._get_problem_type(self.config)
+            if problem_type == 'regression':
+                labels = torch.tensor(labels, dtype=torch.float32)
+            elif problem_type == 'multi_label_classification':
+                one_hot_labels = torch.zeros((len(labels), self.config.num_labels), dtype=torch.float32)
+                for i, label in enumerate(labels):
+                    one_hot_labels[i, label] = 1
+                labels = one_hot_labels
+            else:
+                labels = torch.tensor(labels, dtype=torch.long)
+            res['labels'] = labels
+        return res
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        """
+        Args:
+            batch(`List[Dict[str, Any]]`): The input data in batch
+            padding_to(`int`, optional): Whether padding the batch to a fixed length, if none, the batch
+                will be padded to the `longest`
+        """
+        assert self.tokenizer.pad_token_id is not None
+        padding_side = self.padding_side if self.is_training else 'left'
+        padding_right = padding_side == 'right'
+        packing_mode = self.use_megatron or self._packing and 'position_ids' in batch[0]
+        res = {}
+        if packing_mode:
+            # only support llm
+            for k in ['input_ids', 'labels', 'position_ids', 'loss_scale']:
+                v = self.gather_list(batch, k)
+                if v:
+                    res[k] = [v]
+        else:
+            inputs_embeds = [b['inputs_embeds'] for b in batch if b.get('inputs_embeds') is not None]
+            input_ids = [b['input_ids'] for b in batch if b.get('input_ids') is not None]
+            if inputs_embeds:
+                res['inputs_embeds'] = inputs_embeds
+            if input_ids:
+                res['input_ids'] = input_ids
+            for key in ['labels', 'loss_scale', 'position_ids', 'token_type_ids']:
+                val = [b[key] for b in batch if b.get(key) is not None]
+                if val:
+                    res[key] = val
+
+        keys = [
+            'input_ids', 'inputs_embeds', 'attention_mask', 'labels', 'loss_scale', 'position_ids', 'token_type_ids'
+        ]
+        pad_values = [self.tokenizer.pad_token_id, 0., 0, -100, 0., 0., 0]
+        # Convert to tensor and remove unnecessary dimensions.
+        seq_lens = None
+        for key in keys:
+            if key not in res:
+                continue
+            for i, val in enumerate(res[key]):
+                if isinstance(val, (list, tuple)):
+                    val = torch.tensor(val)
+                elif key == 'inputs_embeds' and val.ndim == 3 or key != 'inputs_embeds' and val.ndim == 2:
+                    val = val[0]
+                res[key][i] = val
+            if not seq_lens:
+                seq_lens = [seq.shape[0] for seq in res[key]]
+        if not packing_mode and seq_lens and ('input_ids' in res or 'inputs_embeds' in res):
+            res['attention_mask'] = [torch.ones(seq_len, dtype=torch.int64) for seq_len in seq_lens]
+            if self.is_training and self.padding_side == 'left':
+                res['position_ids'] = [torch.arange(seq_len, dtype=torch.int64) for seq_len in seq_lens]
+
+        if self.use_megatron:
+            padding_to = math.ceil(max(seq_lens) / 128) * 128
+            cp_size = self.sequence_parallel_size
+            if cp_size > 1:
+                padding_len = padding_to - seq_lens[0]
+                position_ids = res['position_ids'][0].tolist()
+                position_ids += list(range(cp_size * 2)) * (padding_len // (cp_size * 2))
+                res['position_ids'][0] = torch.tensor(position_ids)
+
+        for key, pad_value in zip(keys, pad_values):
+            if key not in res:
+                continue
+            if self.use_megatron and key == 'position_ids' and self.sequence_parallel_size > 1:
+                pass
+            elif padding_to is not None:
+                padding_len = padding_to - seq_lens[0]
+                if padding_len > 0:
+                    res[key][0] = F.pad(res[key][0], (0, padding_len) if padding_right else (padding_len, 0),
+                                        'constant', pad_value)
+            res[key] = self._pad_sequence(res[key], pad_value)
+
+        # multimodal
+        res.update(self._data_collator_mm_data(batch))
+        if not self.use_megatron and (use_torchacc() or self.sequence_parallel_size > 1):
+            res = self._torchacc_xtuner_data_collator(res, padding_to, self.tokenizer, padding_side)
+
+        return res
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        # multimodal
+        res = {}
+        pixel_values = [b['pixel_values'] for b in batch if b.get('pixel_values') is not None]
+        if len(pixel_values) > 0:
+            res['pixel_values'] = torch.concat(pixel_values)
+
+            image_sizes = [b['image_sizes'] for b in batch if b.get('image_sizes') is not None]
+            if len(image_sizes) > 0:
+                res['image_sizes'] = torch.concat(image_sizes)
+
+        pixel_values_videos = [b['pixel_values_videos'] for b in batch if b.get('pixel_values_videos') is not None]
+        if len(pixel_values_videos) > 0:
+            res['pixel_values_videos'] = torch.concat(pixel_values_videos)
+        return res
+
+    def _torchacc_xtuner_data_collator(self, res, padding_to, tokenizer, padding_side):
+        # torchacc & xtuner
+        input_ids = res.get('input_ids')
+        attention_mask = res.get('attention_mask')
+        labels = res.get('labels')
+        loss_scale = res.get('loss_scale')
+        if use_torchacc():
+            from swift.utils.torchacc_utils import pad_and_split_batch
+            rank, _, world_size, _ = get_dist_setting()
+            input_ids, attention_mask, labels, loss_scale = pad_and_split_batch(
+                padding_to,
+                input_ids,
+                attention_mask,
+                labels,
+                loss_scale,
+                self.max_length,
+                tokenizer,
+                rank,
+                world_size,
+                padding_right=padding_side == 'right')
+        if self.sequence_parallel_size > 1 and input_ids is not None:
+            bs, seq_len = input_ids.shape
+            if 'position_ids' not in res:
+                position_ids = torch.arange(seq_len).unsqueeze(0).long().repeat(bs, 1)
+            else:
+                position_ids = res['position_ids']
+            assert padding_side == 'right' or bs == 1, 'Sequence parallel only support padding_side=right'
+            from swift.trainers.sequence_parallel import sequence_parallel
+            if sequence_parallel.world_size() > 1:
+                from swift.trainers.sequence_parallel import sequence_parallel
+                input_ids, _, labels, position_ids, attention_mask, loss_scale = \
+                    sequence_parallel.pad_and_split_inputs(
+                        tokenizer, input_ids, None, labels, position_ids, attention_mask, loss_scale)
+            res['position_ids'] = position_ids
+        _local_var = locals()
+        for key in ['input_ids', 'attention_mask', 'labels', 'loss_scale']:
+            value = _local_var[key]
+            if value is not None:
+                res[key] = value
+        return res
+
+    def print_inputs(self, inputs: Dict[str, Any], tokenizer_kwargs: Optional[Dict[str, Any]] = None) -> None:
+        if tokenizer_kwargs is None:
+            tokenizer_kwargs = {}
+        for key in [
+                'input', 'labels', 'generate', 'chosen_input', 'chosen_labels', 'rejected_input', 'rejected_labels'
+        ]:
+            val = inputs.get(key)  # fix val is a tensor
+            if val is None:
+                val = inputs.get(f'{key}_ids')
+            if val is not None:
+                key_upper = key.upper()
+                logger.info(f'[{key_upper}_IDS] {val}')
+                if key == 'labels' and self.mode in {'seq_cls', 'embedding'}:
+                    continue
+                if isinstance(val, (list, tuple, torch.Tensor)):
+                    val_str = self.safe_decode(val, **tokenizer_kwargs)
+                    logger.info(f'[{key_upper}] {val_str}')
+        if inputs.get('loss_scale') is not None:
+            val = inputs['loss_scale']
+            logger.info(f'[LOSS_SCALE] {val}')
+
+    async def prepare_lmdeploy_pytorch_inputs(self, inputs) -> None:
+        images = inputs.pop('images', None) or []
+        if len(images) == 0:
+            return
+        input_ids = inputs['input_ids']
+        idx_list = findall(input_ids, -100)
+        assert len(idx_list) == len(images), f'len(idx_list): {len(idx_list)}, len(images): {len(images)}'
+        idx_list.insert(0, -1)
+        new_input_ids = []
+        for i in range(len(idx_list) - 1):
+            new_input_ids += input_ids[idx_list[i] + 1:idx_list[i + 1]]
+            images[i]['offset'] = len(new_input_ids)
+            new_input_ids += [images[i]['image_token_id']] * images[i]['image_tokens']
+        new_input_ids += input_ids[idx_list[-1] + 1:]
+        inputs['input_ids'] = new_input_ids
+        inputs['multimodal'] = images
+
+    async def prepare_lmdeploy_turbomind_inputs(self, inputs: Dict[str, Any]) -> None:
+        images = inputs.pop('images', None) or []
+        if len(images) == 0:
+            return
+        from lmdeploy.vl.constants import IMAGE_DUMMY_TOKEN_INDEX
+        input_ids = inputs['input_ids']
+        idx_list = findall(input_ids, -100)
+        assert len(idx_list) == len(images), f'len(idx_list): {len(idx_list)}, len(images): {len(images)}'
+        idx_list.insert(0, -1)
+        new_input_ids = []
+        ranges = []
+        for i in range(len(idx_list) - 1):
+            _range = []
+            new_input_ids += input_ids[idx_list[i] + 1:idx_list[i + 1]]
+            _range.append(len(new_input_ids))
+            new_input_ids += [IMAGE_DUMMY_TOKEN_INDEX] * images[i].shape[0]
+            _range.append(len(new_input_ids))
+            ranges.append(_range)
+        new_input_ids += input_ids[idx_list[-1] + 1:]
+        inputs['input_embeddings'] = [image.to('cpu') for image in images]
+        inputs['input_embedding_ranges'] = ranges
+        inputs['input_ids'] = new_input_ids
+
+    def _pad_sequence(self, sequences: List[torch.Tensor], padding_value: float = 0.) -> torch.Tensor:
+        """Pad sequence by some side
+
+        Args:
+            sequences: The input sequences in tensor.
+            padding_value: The padding value
+
+        Returns:
+            A tensor after padding
+        """
+        padding_side = self.padding_side if self.is_training else 'left'
+        padding_right = padding_side == 'right'
+        if padding_right:
+            return pad_sequence(sequences, batch_first=True, padding_value=padding_value)
+
+        max_len = max([s.shape[0] for s in sequences])
+
+        padded_sequences = []
+        for seq in sequences:
+            pad_length = max_len - seq.shape[0]
+            pad_tuple = [0] * ((seq.dim() - 1) * 2) + [pad_length, 0]
+            padded_seq = F.pad(seq, tuple(pad_tuple), 'constant', padding_value)
+            padded_sequences.append(padded_seq)
+
+        return torch.stack(padded_sequences)
+
+    def safe_decode(self, input_ids: List[int], **tokenizer_kwargs) -> str:
+        if isinstance(self, Template):
+            tokenizer = self.tokenizer
+            placeholder_tokens = self.placeholder_tokens
+        else:
+            tokenizer = self
+            placeholder_tokens = []
+
+        def _is_special(token: int) -> bool:
+            if isinstance(token, float) or token < 0:
+                return True
+            return token in placeholder_tokens
+
+        if isinstance(input_ids, torch.Tensor):
+            input_ids = input_ids.tolist()
+        if len(input_ids) == 0:
+            return ''
+        result_str = ''
+        for i in range(len(input_ids)):
+            if i == 0:
+                if _is_special(input_ids[i]):
+                    s = 0
+                else:
+                    e = 0
+                continue
+            if _is_special(input_ids[i]) and not _is_special(input_ids[i - 1]):
+                s = i
+                result_str += tokenizer.decode(input_ids[e:s], **tokenizer_kwargs)
+            if not _is_special(input_ids[i]) and _is_special(input_ids[i - 1]):
+                e = i
+                result_str += f'[{input_ids[i - 1]} * {e - s}]'
+        if _is_special(input_ids[i]):
+            result_str += f'[{input_ids[i]} * {len(input_ids) - s}]'
+        else:
+            result_str += tokenizer.decode(input_ids[e:], **tokenizer_kwargs)
+        return result_str
+
+    @staticmethod
+    @contextmanager
+    def _patch_flash_attention_forward(modeling_module, position_ids, use_new_func: bool = False):
+        _origin_flash_attention_forward = modeling_module._flash_attention_forward
+
+        def _flash_attention_forward(*args, **kwargs):
+            if use_new_func:
+                from transformers.modeling_flash_attention_utils import (_flash_attention_forward as
+                                                                         flash_attention_forward)
+                if args and isinstance(args[0], nn.Module):
+                    args = args[1:]
+                if 'is_causal' not in kwargs:
+                    kwargs['is_causal'] = True
+            else:
+                flash_attention_forward = _origin_flash_attention_forward
+            kwargs['position_ids'] = position_ids
+            return flash_attention_forward(*args, **kwargs)
+
+        modeling_module._flash_attention_forward = _flash_attention_forward
+        try:
+            yield
+        finally:
+            modeling_module._flash_attention_forward = _origin_flash_attention_forward

ms-swift/swift/llm/template/grounding.py

@@ -0,0 +1,91 @@
+import colorsys
+import itertools
+import os
+from copy import deepcopy
+from typing import Any, List, Literal
+
+import requests
+from modelscope.hub.utils.utils import get_cache_dir
+from PIL import Image, ImageDraw, ImageFont
+
+
+def _shuffle_colors(nums: List[Any]) -> List[Any]:
+    if len(nums) == 1:
+        return nums
+
+    mid = len(nums) // 2
+
+    left = nums[:mid]
+    right = nums[mid:]
+    left = _shuffle_colors(left)
+    right = _shuffle_colors(right)
+    new_nums = []
+    for x, y in zip(left, right):
+        new_nums += [x, y]
+    new_nums += left[len(right):] or right[len(left):]
+    return new_nums
+
+
+def generate_colors():
+    vs_combinations = [(v, s) for v, s in itertools.product([0.7, 0.3, 1], [0.7, 0.3, 1])]
+    colors = [colorsys.hsv_to_rgb(i / 16, s, v) for v, s in vs_combinations for i in _shuffle_colors(list(range(16)))]
+    colors = [(int(r * 255), int(g * 255), int(b * 255)) for r, g, b in colors]
+    return _shuffle_colors(colors)
+
+
+def download_file(url: str) -> str:
+    url = url.rstrip('/')
+    file_name = url.rsplit('/', 1)[-1]
+    cache_dir = os.path.join(get_cache_dir(), 'files')
+    os.makedirs(cache_dir, exist_ok=True)
+    req = requests.get(url)
+    file_path = os.path.join(cache_dir, file_name)
+    with open(file_path, 'wb') as f:
+        f.write(req.content)
+    return file_path
+
+
+colors = generate_colors()
+color_mapping = {}
+
+
+def _calculate_brightness(image, region: List[int]):
+    cropped_image = image.crop(region)
+    grayscale_image = cropped_image.convert('L')
+    pixels = list(grayscale_image.getdata())
+    average_brightness = sum(pixels) / len(pixels)
+    return average_brightness
+
+
+def draw_bbox(image: Image.Image,
+              ref: List[str],
+              bbox: List[List[int]],
+              norm_bbox: Literal['norm1000', 'none'] = 'norm1000'):
+    bbox = deepcopy(bbox)
+    font_path = 'https://modelscope.cn/models/Qwen/Qwen-VL-Chat/resolve/master/SimSun.ttf'
+    # norm bbox
+    for i, box in enumerate(bbox):
+        for i in range(len(box)):
+            box[i] = int(box[i])
+        if norm_bbox == 'norm1000':
+            box[0] = box[0] / 1000 * image.width
+            box[2] = box[2] / 1000 * image.width
+            box[1] = box[1] / 1000 * image.height
+            box[3] = box[3] / 1000 * image.height
+
+    draw = ImageDraw.Draw(image)
+    # draw bbox
+    assert len(ref) == len(bbox), f'len(refs): {len(ref)}, len(bboxes): {len(bbox)}'
+    for (left, top, right, bottom), box_ref in zip(bbox, ref):
+        if box_ref not in color_mapping:
+            color_mapping[box_ref] = colors[len(color_mapping) % len(colors)]
+        color = color_mapping[box_ref]
+        draw.rectangle([(left, top), (right, bottom)], outline=color, width=3)
+    # draw text
+    file_path = download_file(font_path)
+    font = ImageFont.truetype(file_path, 20)
+    for (left, top, _, _), box_ref in zip(bbox, ref):
+        brightness = _calculate_brightness(
+            image, [left, top, min(left + 100, image.width),
+                    min(top + 20, image.height)])
+        draw.text((left, top), box_ref, fill='white' if brightness < 128 else 'black', font=font)

ms-swift/swift/llm/template/register.py

@@ -0,0 +1,57 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Dict, Literal, Optional
+
+from ..utils import Processor
+from .base import Template
+from .template_meta import TemplateMeta
+
+TEMPLATE_MAPPING: Dict[str, TemplateMeta] = {}
+
+
+def register_template(template_meta: TemplateMeta, *, exist_ok: bool = False) -> None:
+    template_type = template_meta.template_type
+    if not exist_ok and template_type in TEMPLATE_MAPPING:
+        raise ValueError(f'The `{template_type}` has already been registered in the TEMPLATE_MAPPING.')
+    TEMPLATE_MAPPING[template_type] = template_meta
+
+
+def get_template(
+        template_type: str,
+        processor: Processor,
+        default_system: Optional[str] = None,
+        max_length: Optional[int] = None,
+        *,
+        use_chat_template: bool = True,
+        template_backend: Literal['swift', 'jinja'] = 'swift',
+        truncation_strategy: Literal['raise', 'left', 'right'] = 'raise',
+        max_pixels: Optional[int] = None,  # h * w
+        agent_template: Optional[str] = None,
+        norm_bbox: Literal['norm1000', 'none', None] = None,
+        response_prefix: Optional[str] = None,
+        # train
+        padding_side: Literal['left', 'right'] = 'right',
+        loss_scale: str = 'default',
+        sequence_parallel_size: int = 1) -> 'Template':
+    template_meta = TEMPLATE_MAPPING[template_type]
+    template_cls = template_meta.template_cls
+    return template_cls(
+        processor,
+        template_meta,
+        default_system,
+        max_length,
+        use_chat_template=use_chat_template,
+        template_backend=template_backend,
+        truncation_strategy=truncation_strategy,
+        max_pixels=max_pixels,
+        agent_template=agent_template,
+        norm_bbox=norm_bbox,
+        response_prefix=response_prefix,
+        padding_side=padding_side,
+        loss_scale=loss_scale,
+        sequence_parallel_size=sequence_parallel_size,
+    )
+
+
+def get_template_meta(template_type: str) -> TemplateMeta:
+    return TEMPLATE_MAPPING[template_type]

ms-swift/swift/llm/template/template/__init__.py

@@ -0,0 +1,2 @@
+from . import (deepseek, emu3, gemma, glm, idefics3, internlm, internvl, llama, llava, llm, megrez, microsoft, minicpm,
+               minimax, mistral, molmo, moonshot, mplug, openbuddy, pixtral, qwen, stepfun, valley, yi)