Add files using upload-large-folder tool

swift/llm/dataset/preprocessor/extra.py

@@ -0,0 +1,112 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+
+from .core import ResponsePreprocessor
+
+
+class GroundingMixin:
+    """This class offers prompts to the grounding task"""
+    task_type: Optional[str] = None
+
+    _grounding_language_mixin = [0.8, 0.2]
+    _grounding_prompts = {
+        'grounding': {
+            'en': [('<ref-object>', '<bbox>'), ('The positions of <ref-object> is', '<bbox>'),
+                   ('Find the positions of <ref-object>', '<bbox>'), ('Where is <ref-object>', '<bbox>'),
+                   ('Find <ref-object>', '<bbox>'), ('Show me <ref-object>', '<bbox>'),
+                   ('Detect <ref-object>', '<bbox>'), ('Locate <ref-object>', '<bbox>'),
+                   ('Tell me the location of <ref-object>', '<bbox>'), ('Give the location of <ref-object>', '<bbox>'),
+                   ('Provide the bounding box coordinate of <ref-object>', '<bbox>')],
+            'zh': [('<ref-object>', '<bbox>'), ('<ref-object>的位置在图片中', '<bbox>'), ('<ref-object>在图片中', '<bbox>'),
+                   ('<ref-object>在', '<bbox>'), ('找到<ref-object>的位置', '<bbox>'), ('<ref-object>在哪里', '<bbox>'),
+                   ('提供<ref-object>的坐标位置', '<bbox>')]
+        },
+        'caption': {
+            'en': [
+                ('<bbox>', '<ref-object>'),
+                ('The object at position <bbox>', '<ref-object>'),
+                ('This <bbox> is', '<ref-object>'),
+                ('What is the object at <bbox>', '<ref-object>'),
+                ('Describe <bbox>', '<ref-object>'),
+                ('<bbox> is', '<ref-object>'),
+                ('The bounding box coordinate <bbox> contains', '<ref-object>'),
+            ],
+            'zh': [
+                ('<bbox>', '<ref-object>'),
+                ('<bbox>是什么', '<ref-object>'),
+                ('<bbox>的位置包含', '<ref-object>'),
+                ('描述<bbox>', '<ref-object>'),
+                ('<bbox>中是', '<ref-object>'),
+                ('坐标<bbox>描述了什么', '<ref-object>'),
+                ('描述<bbox>中的事物', '<ref-object>'),
+            ]
+        },
+    }
+
+    def construct_grounding_prompt(self):
+        # TODO Only support one bbox to one object
+        lang = np.random.choice(['en', 'zh'], p=[0.8, 0.2])
+        prompts = GroundingMixin._grounding_prompts[self.task_type][lang]
+        query, response = prompts[np.random.choice(range(len(prompts)))]
+        return query, response
+
+
+class TextGenerationPreprocessor(ResponsePreprocessor):
+
+    def __init__(self,
+                 *,
+                 prompt: str,
+                 query_tag: str = '{{QUERY}}',
+                 columns: Optional[Dict[str, str]] = None,
+                 **kwargs) -> None:
+        self.query_tag = query_tag
+        self.prompt = prompt
+        super().__init__(columns=columns, **kwargs)
+
+    def preprocess(self, row: Dict[str, Any]) -> Dict[str, Any]:
+        row['query'] = self.prompt.replace(self.query_tag, row['query'])
+        return super().preprocess(row)
+
+
+class ClsGenerationPreprocessor(ResponsePreprocessor):
+
+    def __init__(self,
+                 labels: List[str],
+                 *,
+                 task: str,
+                 is_pair_seq: bool = False,
+                 columns: Optional[Dict[str, str]] = None,
+                 **kwargs) -> None:
+        self.labels = labels
+        self.task = task
+        self.is_pair_seq = is_pair_seq
+
+        category = ', '.join(labels)
+        self.sentence2_key = 'sentence2'
+        self.label_key = 'label'
+        if is_pair_seq:
+            self.sentence_key = 'sentence1'
+            inputs = 'Sentence1: {sentence1}\nSentence2: {sentence2}'
+        else:
+            self.sentence_key = 'sentence'
+            inputs = 'Sentence: {sentence}'
+        self.prompt = f"""Task: {task}
+{inputs}
+Category: {category}
+Output:"""
+        super().__init__(columns=columns, **kwargs)
+
+    def preprocess(self, row: Dict[str, Any]) -> Optional[Dict[str, Any]]:
+        label = row.pop(self.label_key, None)
+        if label is None:
+            return
+
+        if self.is_pair_seq:
+            query = self.prompt.format(sentence1=row.pop(self.sentence_key), sentence2=row.pop(self.sentence2_key))
+        else:
+            query = self.prompt.format(sentence=row.pop(self.sentence_key))
+        row['query'] = query
+        row['response'] = self.labels[int(label)]
+        return super().preprocess(row)

swift/llm/model/model/microsoft.py

@@ -0,0 +1,234 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from functools import partial
+from types import MethodType
+from typing import Any, Dict
+
+from transformers import AutoConfig
+
+from swift.llm import TemplateType
+from swift.utils import get_device, get_env_args
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..patcher import patch_ignore_check_imports, patch_output_clone
+from ..register import (Model, ModelGroup, ModelMeta, get_model_tokenizer_multimodal,
+                        get_model_tokenizer_with_flash_attn, register_model)
+from ..utils import ModelInfo, use_submodel_func
+
+
+def get_model_tokenizer_phi3_vision(model_dir: str,
+                                    model_info: ModelInfo,
+                                    model_kwargs: Dict[str, Any],
+                                    load_model: bool = True,
+                                    **kwargs):
+    processor_kwargs = {}
+    if 'num_crops' in kwargs:
+        processor_kwargs['num_crops'] = get_env_args('num_crops', int, kwargs['num_crops'])
+    from transformers import AutoProcessor
+    processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True, **processor_kwargs)
+    model, tokenizer = get_model_tokenizer_with_flash_attn(
+        model_dir, model_info, model_kwargs, load_model, tokenizer=processor.tokenizer, **kwargs)
+
+    if load_model:
+        patch_output_clone(model.model.vision_embed_tokens.wte)
+
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.phi3_vision,
+        [
+            ModelGroup([
+                Model('LLM-Research/Phi-3-vision-128k-instruct', 'microsoft/Phi-3-vision-128k-instruct'),
+                Model('LLM-Research/Phi-3.5-vision-instruct', 'microsoft/Phi-3.5-vision-instruct'),
+            ])
+        ],
+        TemplateType.phi3_vision,
+        partial(get_model_tokenizer_phi3_vision, num_crops=4),
+        architectures=['Phi3VForCausalLM'],
+        model_arch=ModelArch.phi3_vision,
+        requires=['transformers>=4.36'],
+        tags=['vision'],
+    ))
+
+
+def get_model_tokenizer_phi4_multimodal(*args, **kwargs):
+    model, processor = get_model_tokenizer_multimodal(*args, **kwargs)
+    processor.audio_processor.audio_compression_rate = processor.audio_processor.compression_rate
+    processor.audio_processor.audio_downsample_rate = processor.audio_processor.qformer_compression_rate
+    processor.audio_processor.audio_feat_stride = processor.audio_processor.feat_stride
+    del processor.audio_processor.feature_size
+    del processor.audio_processor.sampling_rate
+    del processor.audio_processor.padding_value
+    del processor.__class__.chat_template
+    processor.chat_template = None
+    if model is not None:
+        model.set_lora_adapter(['vision', 'speech'])
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.phi4_multimodal,
+        [ModelGroup([
+            Model('LLM-Research/Phi-4-multimodal-instruct', 'microsoft/Phi-4-multimodal-instruct'),
+        ])],
+        TemplateType.phi4_multimodal,
+        get_model_tokenizer_phi4_multimodal,
+        architectures=['Phi4MMForCausalLM'],
+        model_arch=ModelArch.phi4_multimodal,
+        requires=['transformers>=4.36,<4.49', 'backoff', 'soundfile'],
+        tags=['vision', 'audio'],
+    ))
+
+
+def get_model_tokenizer_florence(model_dir: str,
+                                 model_info: ModelInfo,
+                                 model_kwargs: Dict[str, Any],
+                                 load_model: bool = True,
+                                 **kwargs):
+    model_config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    model_config.vision_config.model_type = 'davit'  # fix merge-lora
+    if model_kwargs['device_map'] == 'auto':
+        model_kwargs['device_map'] = get_device()
+    kwargs['model_config'] = model_config
+    with patch_ignore_check_imports():
+        model, processor = get_model_tokenizer_multimodal(model_dir, model_info, model_kwargs, load_model, **kwargs)
+
+    if model is not None:
+        model.vision_tower.enable_checkpoint = True
+        use_submodel_func(model, 'language_model', ['generate', 'forward'])
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.florence,
+        [
+            # llama2
+            ModelGroup([
+                Model('AI-ModelScope/Florence-2-base-ft', 'microsoft/Florence-2-base-ft'),
+                Model('AI-ModelScope/Florence-2-base', 'microsoft/Florence-2-base'),
+                Model('AI-ModelScope/Florence-2-large', 'microsoft/Florence-2-large'),
+                Model('AI-ModelScope/Florence-2-large-ft', 'microsoft/Florence-2-large-ft'),
+            ]),
+        ],
+        TemplateType.florence,
+        get_model_tokenizer_florence,
+        architectures=['Florence2ForConditionalGeneration'],
+        model_arch=ModelArch.florence,
+        tags=['vision'],
+    ))
+
+
+def get_model_tokenizer_phi3_small(model_dir: str,
+                                   model_info: ModelInfo,
+                                   model_kwargs: Dict[str, Any],
+                                   load_model: bool = True,
+                                   **kwargs):
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+
+    def rotary_emb(self, query_states, key_states, **kwargs):
+        q_type = query_states.dtype
+        k_type = key_states.dtype
+        query_states, key_states = self.rotory_emb_origin(query_states, key_states, **kwargs)
+        query_states = query_states.to(q_type)
+        key_states = key_states.to(k_type)
+        return query_states, key_states
+
+    if model is not None:
+        for i in range(32):
+            re = model.model.layers[i].self_attn.rotary_emb
+            re.rotory_emb_origin = re.forward
+            re.forward = MethodType(rotary_emb, re)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi3_small,
+        [
+            ModelGroup([
+                Model('LLM-Research/Phi-3-small-8k-instruct', 'microsoft/Phi-3-small-8k-instruct'),
+                Model('LLM-Research/Phi-3-small-128k-instruct', 'microsoft/Phi-3-small-128k-instruct'),
+            ]),
+        ],
+        TemplateType.phi3,
+        get_model_tokenizer_phi3_small,
+        architectures=['Phi3SmallForCausalLM'],
+        model_arch=ModelArch.phi3_small,
+        requires=['transformers>=4.36'],
+    ))
+
+
+def get_model_tokenizer_phi(model_dir: str,
+                            model_info: ModelInfo,
+                            model_kwargs: Dict[str, Any],
+                            load_model: bool = True,
+                            **kwargs):
+    # TODO: check
+    return get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi2,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/phi-2', 'microsoft/phi-2'),
+            ]),
+        ],
+        TemplateType.default,
+        get_model_tokenizer_phi,
+        architectures=['PhiForCausalLM'],
+        model_arch=ModelArch.phi2,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi3,
+        [
+            ModelGroup([
+                Model('LLM-Research/Phi-3-mini-4k-instruct', 'microsoft/Phi-3-mini-4k-instruct'),
+                Model('LLM-Research/Phi-3-mini-128k-instruct', 'microsoft/Phi-3-mini-128k-instruct'),
+                Model('LLM-Research/Phi-3-medium-4k-instruct', 'microsoft/Phi-3-medium-4k-instruct'),
+                Model('LLM-Research/Phi-3-medium-128k-instruct', 'microsoft/Phi-3-medium-128k-instruct'),
+                Model('LLM-Research/Phi-3.5-mini-instruct', 'microsoft/Phi-3.5-mini-instruct'),
+            ]),
+            ModelGroup(Model('LLM-Research/Phi-4-mini-instruct', 'microsoft/Phi-4-mini-instruct'))
+        ],
+        TemplateType.phi3,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Phi3ForCausalLM'],
+        requires=['transformers>=4.36'],
+        model_arch=ModelArch.phi3,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi4,
+        [
+            ModelGroup([
+                Model('LLM-Research/phi-4', 'microsoft/phi-4'),
+            ]),
+        ],
+        TemplateType.phi4,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Phi3ForCausalLM'],
+        requires=['transformers>=4.36'],
+        model_arch=ModelArch.phi3,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.phi3_moe,
+        [
+            ModelGroup([
+                Model('LLM-Research/Phi-3.5-MoE-instruct', 'microsoft/Phi-3.5-MoE-instruct'),
+            ]),
+        ],
+        TemplateType.phi3,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['PhiMoEForCausalLM'],
+        requires=['transformers>=4.36'],
+    ))

swift/llm/model/model/minicpm.py

@@ -0,0 +1,207 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from functools import partial
+from types import MethodType
+from typing import Any, Dict
+
+from transformers import AutoConfig
+from transformers.utils import strtobool
+
+from swift.llm import TemplateType
+from swift.utils import get_env_args
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..patcher import patch_device_map, patch_fixed_device, patch_output_clone
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import ModelInfo, use_submodel_func
+from .deepseek import get_model_tokenizer_deepseek_moe
+
+register_model(
+    ModelMeta(
+        LLMModelType.minicpm_moe,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-MoE-8x2B', 'openbmb/MiniCPM-MoE-8x2B'),
+            ]),
+        ],
+        TemplateType.minicpm,
+        get_model_tokenizer_deepseek_moe,
+        architectures=['MiniCPMForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.36'],
+    ))
+
+
+def _patch_minicpmv_device_map(model) -> None:
+    if not hasattr(model, 'hf_device_map') or len(model.hf_device_map.values()) == 1:
+        return
+
+    device = list(model.hf_device_map.values())[0]
+    if hasattr(model, 'get_vision_embedding') and not hasattr(model, '_old_get_vision_embedding'):
+        # minicpm-v-v2-chat; avoid double patching
+        _old_get_vision_embedding = model.__class__.get_vision_embedding
+
+        def _get_vision_embedding(self, pixel_values):
+            if len(pixel_values) == 0:
+                return _old_get_vision_embedding(self, pixel_values)
+            output = _old_get_vision_embedding(self, pixel_values)
+            return output.to(device=device)
+
+        model.__class__._old_get_vision_embedding = _old_get_vision_embedding
+        model.__class__.get_vision_embedding = _get_vision_embedding
+
+    if hasattr(model, 'resampler'):  # minicpm-v-v2_5-chat
+        patch_fixed_device(model.resampler, device)
+
+
+def get_model_tokenizer_minicpmv(model_dir: str,
+                                 model_info: ModelInfo,
+                                 model_kwargs: Dict[str, Any],
+                                 load_model: bool = True,
+                                 **kwargs):
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if load_model:
+        model.resampler.to(model_info.torch_dtype)  # fix float32
+        _patch_minicpmv_device_map(model)
+        func_list = ['generate', 'get_input_embeddings', 'forward']
+        use_submodel_func(model, 'llm', func_list)
+        if hasattr(model, 'get_slice_image_placeholder'):
+            tokenizer.get_slice_image_placeholder = MethodType(model.get_slice_image_placeholder, tokenizer)
+            tokenizer.transform = MethodType(model.transform, tokenizer)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmv,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-V', 'openbmb/MiniCPM-V'),
+                Model('OpenBMB/MiniCPM-V-2', 'openbmb/MiniCPM-V-2'),
+            ], ),
+        ],
+        TemplateType.minicpmv,
+        get_model_tokenizer_minicpmv,
+        architectures=['MiniCPMV'],
+        model_arch=ModelArch.minicpmv,
+        requires=['timm', 'transformers<4.42'],
+        tags=['vision'],
+    ))
+
+
+def get_model_tokenizer_minicpmv_2_x(model_dir: str,
+                                     model_info: ModelInfo,
+                                     model_kwargs: Dict[str, Any],
+                                     load_model: bool = True,
+                                     **kwargs):
+    from transformers import AutoProcessor
+    processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+    version = kwargs.get('version')
+    if version == 'o2.6':
+        model_config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+        model_config.init_tts = strtobool(get_env_args('init_tts', str, 'false'))
+        model_config.init_audio = strtobool(get_env_args('init_audio', str, 'false'))
+        kwargs['model_config'] = model_config
+    with patch_device_map():
+        model, tokenizer = get_model_tokenizer_minicpmv(
+            model_dir, model_info, model_kwargs, load_model, tokenizer=processor.tokenizer, **kwargs)
+    if load_model:
+        embedding = model.get_input_embeddings()
+        patch_output_clone(embedding)
+
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmv2_5,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-Llama3-V-2_5', 'openbmb/MiniCPM-Llama3-V-2_5'),
+            ], ),
+        ],
+        TemplateType.minicpmv2_5,
+        get_model_tokenizer_minicpmv_2_x,
+        architectures=['MiniCPMV'],
+        model_arch=ModelArch.minicpmv,
+        requires=['timm', 'transformers>=4.36'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmv2_6,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-V-2_6', 'openbmb/MiniCPM-V-2_6'),
+            ], ),
+        ],
+        TemplateType.minicpmv2_6,
+        get_model_tokenizer_minicpmv_2_x,
+        architectures=['MiniCPMV'],
+        model_arch=ModelArch.minicpmv,
+        requires=['timm', 'transformers>=4.36', 'decord'],
+        tags=['vision', 'video'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minicpmo2_6,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-o-2_6', 'openbmb/MiniCPM-o-2_6'),
+            ]),
+        ],
+        TemplateType.minicpmo2_6,
+        partial(get_model_tokenizer_minicpmv_2_x, version='o2.6'),
+        architectures=['MiniCPMO'],
+        model_arch=ModelArch.minicpmv,
+        requires=['timm', 'transformers>=4.36', 'decord', 'soundfile'],
+        tags=['vision', 'video', 'omni', 'audio'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.minicpm,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-2B-sft-fp32', 'openbmb/MiniCPM-2B-sft-fp32'),
+                Model('OpenBMB/MiniCPM-2B-dpo-fp32', 'openbmb/MiniCPM-2B-dpo-fp32'),
+                Model('OpenBMB/MiniCPM-1B-sft-bf16', 'openbmb/MiniCPM-1B-sft-bf16'),
+            ], ),
+        ],
+        TemplateType.minicpm,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MiniCPMForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.36.0'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.minicpm_chatml,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM-2B-128k', 'openbmb/MiniCPM-2B-128k'),
+            ]),
+        ],
+        TemplateType.chatml,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MiniCPMForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.36'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.minicpm3,
+        [
+            ModelGroup([
+                Model('OpenBMB/MiniCPM3-4B', 'openbmb/MiniCPM3-4B'),
+            ]),
+        ],
+        TemplateType.chatml,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MiniCPM3ForCausalLM'],
+        model_arch=ModelArch.deepseek_v2,
+        requires=['transformers>=4.36'],
+    ))

swift/llm/model/model/minimax.py

@@ -0,0 +1,156 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from typing import Any, Dict
+
+import json
+from transformers import AutoConfig, AutoProcessor
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from swift.llm import TemplateType
+from swift.utils import get_device, get_device_count, get_dist_setting, get_logger
+from ..constant import LLMModelType, MLLMModelType
+from ..patcher import patch_ignore_check_imports
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import ModelInfo
+
+logger = get_logger()
+
+
+def get_model_tokenizer_minimax_vl(model_dir: str,
+                                   model_info: ModelInfo,
+                                   model_kwargs: Dict[str, Any],
+                                   load_model: bool = True,
+                                   **kwargs):
+    logger.warn('NOTE: minimax-vl-01 model does not support training.')
+    n_gpu = get_device_count()
+    _, local_rank, _, local_world_size = get_dist_setting()
+    device_ids = list(range(max(local_rank, 0), n_gpu, local_world_size))
+    config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    kwargs['model_config'] = config
+    if kwargs.get('attn_impl') == 'flash_attn':
+        config.attn_type_list = [1] * len(config.attn_type_list)
+    else:
+        config.attn_type_list = [0] * len(config.attn_type_list)
+    if 'quantization_config' in model_kwargs:
+        quantization_config = model_kwargs['quantization_config']
+        from transformers import QuantoConfig
+        if isinstance(quantization_config, QuantoConfig):
+            quantization_config.modules_to_not_convert = (
+                [
+                    'vision_tower',
+                    'image_newline',
+                    'multi_modal_projector',
+                    'lm_head',
+                    'embed_tokens',
+                ] + [f'model.layers.{i}.coefficient' for i in range(config.text_config.num_hidden_layers)]
+                + [f'model.layers.{i}.block_sparse_moe.gate' for i in range(config.text_config.num_hidden_layers)])
+
+    if len(device_ids) > 1:
+        model_safetensors_index_path = os.path.join(model_dir, 'model.safetensors.index.json')
+        with open(model_safetensors_index_path, 'r') as f:
+            model_safetensors_index = json.load(f)
+        weight_map = model_safetensors_index['weight_map']
+        vision_map = {}
+        for key, value in weight_map.items():
+            if 'vision_tower' in key or 'image_newline' in key or 'multi_modal_projector' in key:
+                new_key = key.replace('.weight', '').replace('.bias', '')
+                if new_key not in vision_map:
+                    vision_map[new_key] = value
+
+        device_map = {
+            'language_model.model.embed_tokens': get_device(device_ids[0]),
+            'language_model.model.norm': get_device(device_ids[len(device_ids) - 1]),
+            'language_model.lm_head': get_device(device_ids[len(device_ids) - 1])
+        }
+        for key, value in vision_map.items():
+            device_map[key] = get_device(device_ids[0])
+        device_map['vision_tower.vision_model.post_layernorm'] = get_device(device_ids[0])
+        layers_per_device = config.text_config.num_hidden_layers // len(device_ids)
+        for i in range(len(device_ids)):
+            for j in range(layers_per_device):
+                device_map[f'language_model.model.layers.{i * layers_per_device + j}'] = get_device(device_ids[i])
+        model_kwargs['device_map'] = device_map
+
+    MiniMaxVL01ProcessorKwargs = get_class_from_dynamic_module('processing_minimax_vl_01.MiniMaxVL01ProcessorKwargs',
+                                                               model_dir)
+    get_hw_multiple_of = get_class_from_dynamic_module('processing_minimax_vl_01.get_hw_multiple_of', model_dir)
+    get_num_token = get_class_from_dynamic_module('processing_minimax_vl_01.get_num_token', model_dir)
+
+    processor = AutoProcessor.from_pretrained(model_dir, trust_remote_code=True)
+    processor.MiniMaxVL01ProcessorKwargs = MiniMaxVL01ProcessorKwargs
+    processor.get_hw_multiple_of = get_hw_multiple_of
+    processor.get_num_token = get_num_token
+    with patch_ignore_check_imports():
+        model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model,
+                                                               **kwargs)
+    processor.tokenizer = tokenizer
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.minimax_vl, [
+            ModelGroup([
+                Model('MiniMax/MiniMax-VL-01', 'MiniMaxAI/MiniMax-VL-01'),
+            ]),
+        ],
+        TemplateType.minimax_vl,
+        get_model_tokenizer_minimax_vl,
+        architectures=['MiniMaxVL01ForConditionalGeneration'],
+        tags=['vision']))
+
+
+def get_model_tokenizer_minimax_text(model_dir: str,
+                                     model_info: ModelInfo,
+                                     model_kwargs: Dict[str, Any],
+                                     load_model: bool = True,
+                                     **kwargs):
+    logger.warn('NOTE: minimax-text-01 model does not support training.')
+    n_gpu = get_device_count()
+    _, local_rank, _, local_world_size = get_dist_setting()
+    device_ids = list(range(max(local_rank, 0), n_gpu, local_world_size))
+    config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    kwargs['model_config'] = config
+    if kwargs.get('attn_impl') == 'flash_attn':
+        config.attn_type_list = [1] * len(config.attn_type_list)
+    else:
+        config.attn_type_list = [0] * len(config.attn_type_list)
+    if 'quantization_config' in model_kwargs:
+        quantization_config = model_kwargs['quantization_config']
+        from transformers import QuantoConfig
+        if isinstance(quantization_config, QuantoConfig):
+            quantization_config.modules_to_not_convert = (
+                [
+                    'lm_head',
+                    'embed_tokens',
+                ] + [f'model.layers.{i}.coefficient' for i in range(config.num_hidden_layers)]
+                + [f'model.layers.{i}.block_sparse_moe.gate' for i in range(config.num_hidden_layers)])
+
+    if len(device_ids) > 1:
+        layers_per_device = config.num_hidden_layers // len(device_ids)
+        # set device map
+        device_map = {
+            'model.embed_tokens': get_device(0),
+            'model.norm': get_device(len(device_ids) - 1),
+            'lm_head': get_device(len(device_ids) - 1)
+        }
+        for i in range(len(device_ids)):
+            for j in range(layers_per_device):
+                device_map[f'model.layers.{i * layers_per_device + j}'] = get_device(i)
+        model_kwargs['device_map'] = device_map
+    with patch_ignore_check_imports():
+        model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model,
+                                                               **kwargs)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.minimax, [
+            ModelGroup([
+                Model('MiniMax/MiniMax-Text-01', 'MiniMaxAI/MiniMax-Text-01'),
+            ]),
+        ],
+        TemplateType.minimax,
+        get_model_tokenizer_minimax_text,
+        architectures=['MiniMaxText01ForCausalLM']))

swift/llm/model/model/mistral.py

@@ -0,0 +1,157 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict
+
+from swift.llm import TemplateType
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..register import (Model, ModelGroup, ModelMeta, get_model_tokenizer_multimodal,
+                        get_model_tokenizer_with_flash_attn, register_model)
+from ..utils import ModelInfo
+
+register_model(
+    ModelMeta(
+        LLMModelType.mistral,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Mistral-7B-Instruct-v0.1', 'mistralai/Mistral-7B-Instruct-v0.1'),
+                Model('AI-ModelScope/Mistral-7B-Instruct-v0.2', 'mistralai/Mistral-7B-Instruct-v0.2'),
+                Model('LLM-Research/Mistral-7B-Instruct-v0.3', 'mistralai/Mistral-7B-Instruct-v0.3'),
+                Model('AI-ModelScope/Mistral-7B-v0.1', 'mistralai/Mistral-7B-v0.1'),
+                Model('AI-ModelScope/Mistral-7B-v0.2-hf', 'alpindale/Mistral-7B-v0.2-hf'),
+            ]),
+            ModelGroup([
+                Model('swift/Codestral-22B-v0.1', 'mistralai/Codestral-22B-v0.1'),
+            ]),
+        ],
+        TemplateType.llama,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MistralForCausalLM'],
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.34'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.mixtral, [
+            ModelGroup([
+                Model('AI-ModelScope/Mixtral-8x7B-Instruct-v0.1', 'mistralai/Mixtral-8x7B-Instruct-v0.1'),
+                Model('AI-ModelScope/Mixtral-8x7B-v0.1', 'mistralai/Mixtral-8x7B-v0.1'),
+                Model('AI-ModelScope/Mixtral-8x22B-v0.1', 'mistral-community/Mixtral-8x22B-v0.1'),
+            ],
+                       requires=['transformers>=4.36']),
+            ModelGroup([
+                Model('AI-ModelScope/Mixtral-8x7b-AQLM-2Bit-1x16-hf', 'ISTA-DASLab/Mixtral-8x7b-AQLM-2Bit-1x16-hf'),
+            ],
+                       requires=['transformers>=4.38', 'aqlm', 'torch>=2.2.0']),
+        ],
+        TemplateType.llama,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MixtralForCausalLM'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.mistral_nemo, [
+            ModelGroup([
+                Model('AI-ModelScope/Mistral-Small-Instruct-2409', 'mistralai/Mistral-Small-Instruct-2409'),
+                Model('LLM-Research/Mistral-Large-Instruct-2407', 'mistralai/Mistral-Large-Instruct-2407'),
+                Model('AI-ModelScope/Mistral-Nemo-Base-2407', 'mistralai/Mistral-Nemo-Base-2407'),
+                Model('AI-ModelScope/Mistral-Nemo-Instruct-2407', 'mistralai/Mistral-Nemo-Instruct-2407'),
+            ],
+                       requires=['transformers>=4.43']),
+            ModelGroup([
+                Model('AI-ModelScope/Ministral-8B-Instruct-2410', 'mistralai/Ministral-8B-Instruct-2410'),
+            ],
+                       requires=['transformers>=4.46']),
+        ],
+        TemplateType.mistral_nemo,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MistralForCausalLM'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.mistral_2501, [
+            ModelGroup([
+                Model('mistralai/Mistral-Small-24B-Base-2501', 'mistralai/Mistral-Small-24B-Base-2501'),
+                Model('mistralai/Mistral-Small-24B-Instruct-2501', 'mistralai/Mistral-Small-24B-Instruct-2501'),
+            ]),
+        ],
+        TemplateType.mistral_2501,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MistralForCausalLM'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.zephyr,
+        [
+            ModelGroup([
+                Model('modelscope/zephyr-7b-beta', 'HuggingFaceH4/zephyr-7b-beta'),
+            ]),
+        ],
+        TemplateType.zephyr,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['MistralForCausalLM'],
+        requires=['transformers>=4.34'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.wizardlm2_moe,
+        [ModelGroup([
+            Model('AI-ModelScope/WizardLM-2-8x22B', 'alpindale/WizardLM-2-8x22B'),
+        ])],
+        TemplateType.wizardlm2_moe,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MixtralForCausalLM'],
+        requires=['transformers>=4.36'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.wizardlm2,
+        [ModelGroup([
+            Model('AI-ModelScope/WizardLM-2-7B-AWQ', 'MaziyarPanahi/WizardLM-2-7B-AWQ'),
+        ])],
+        TemplateType.wizardlm2,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MistralForCausalLM'],
+        requires=['transformers>=4.34'],
+    ))
+
+
+def get_model_tokenizer_mistral_2503(model_dir: str,
+                                     model_info: ModelInfo,
+                                     model_kwargs: Dict[str, Any],
+                                     load_model: bool = True,
+                                     **kwargs):
+    try:
+        from transformers import Mistral3ForConditionalGeneration
+    except ImportError:
+        raise ImportError('Please install Gemma3ForConditionalGeneration by running '
+                          '`pip install git+https://github.com/huggingface/transformers@v4.49.0-Mistral-3`')
+
+    kwargs['automodel_class'] = kwargs['automodel_class'] or Mistral3ForConditionalGeneration
+    model, processor = get_model_tokenizer_multimodal(model_dir, model_info, model_kwargs, load_model, **kwargs)
+
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mistral_2503,
+        [
+            ModelGroup([
+                Model('mistralai/Mistral-Small-3.1-24B-Base-2503', 'mistralai/Mistral-Small-3.1-24B-Base-2503'),
+                Model('mistralai/Mistral-Small-3.1-24B-Instruct-2503', 'mistralai/Mistral-Small-3.1-24B-Instruct-2503'),
+            ]),
+        ],
+        TemplateType.mistral_2503,
+        get_model_tokenizer_mistral_2503,
+        architectures=['Mistral3ForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers>=4.49'],
+    ), )

swift/llm/model/model/mllm.py

@@ -0,0 +1,180 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from types import MethodType
+from typing import Any, Dict
+
+import torch
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from swift.llm import TemplateType
+from swift.llm.model.model.qwen import get_model_tokenizer_qwen2_vl
+from swift.utils import get_logger
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..patcher import patch_output_clone, patch_output_normalizer
+from ..register import (Model, ModelGroup, ModelMeta, get_model_tokenizer_multimodal,
+                        get_model_tokenizer_with_flash_attn, register_model)
+from ..utils import ModelInfo, use_submodel_func
+
+logger = get_logger()
+
+
+def get_model_tokenizer_idefics(model_dir: str, *args, **kwargs):
+    from transformers import AutoModelForVision2Seq
+    kwargs['automodel_class'] = kwargs['automodel_class'] or AutoModelForVision2Seq
+    model, processor = get_model_tokenizer_multimodal(model_dir, *args, **kwargs)
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.idefics3,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Idefics3-8B-Llama3', 'HuggingFaceM4/Idefics3-8B-Llama3'),
+            ]),
+        ],
+        TemplateType.idefics3,
+        get_model_tokenizer_idefics,
+        model_arch=ModelArch.idefics3,
+        architectures=['Idefics3ForConditionalGeneration'],
+        tags=['vision'],
+        requires=['transformers>=4.45'],
+    ))
+
+
+def get_model_tokenizer_pixtral(model_dir: str, *args, **kwargs):
+    from transformers import LlavaForConditionalGeneration
+    kwargs['automodel_class'] = kwargs['automodel_class'] or LlavaForConditionalGeneration
+    model, processor = get_model_tokenizer_multimodal(model_dir, *args, **kwargs)
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.pixtral,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/pixtral-12b', 'mistral-community/pixtral-12b'),
+            ]),
+        ],
+        TemplateType.pixtral,
+        get_model_tokenizer_pixtral,
+        model_arch=ModelArch.llava_hf,
+        architectures=['LlavaForConditionalGeneration'],
+        requires=['transformers>=4.45'],
+        tags=['vision'],
+    ))
+
+
+def get_model_tokenizer_molmoe(model_dir: str,
+                               model_info: ModelInfo,
+                               model_kwargs: Dict[str, Any],
+                               load_model: bool = True,
+                               **kwargs):
+    model, processor = get_model_tokenizer_multimodal(model_dir, model_info, model_kwargs, load_model, **kwargs)
+
+    # fix bug for molmoe-1b
+    def to_dict(self, *args, **kwargs):
+        res = self._to_dict(*args, **kwargs)
+        res['vision_backbone'] = self.vision_backbone.__dict__
+        res.pop('to_dict')
+        res.pop('_to_dict')
+        return res
+
+    if model is not None:
+        model.config._to_dict = model.config.to_dict
+        model.config.to_dict = MethodType(to_dict, model.config)
+        patch_output_clone(model.model.transformer.wte)
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.molmoe,
+        [
+            ModelGroup([
+                Model('LLM-Research/MolmoE-1B-0924', 'allenai/MolmoE-1B-0924'),
+            ]),
+        ],
+        TemplateType.molmo,
+        get_model_tokenizer_molmoe,
+        model_arch=ModelArch.molmo,
+        torch_dtype=torch.float32,
+        architectures=['OLMoForCausalLM'],
+        tags=['vision'],
+        requires=['transformers>=4.45'],
+    ))
+
+
+def get_model_tokenizer_molmo(model_dir: str,
+                              model_info: ModelInfo,
+                              model_kwargs: Dict[str, Any],
+                              load_model: bool = True,
+                              **kwargs):
+    model_cls = get_class_from_dynamic_module('modeling_molmo.MolmoForCausalLM', model_dir)
+    model_cls._no_split_modules = ['MolmoSequentialBlock']
+    model, processor = get_model_tokenizer_multimodal(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if model is not None:
+        patch_output_clone(model.model.transformer.wte)
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.molmo,
+        [
+            ModelGroup([
+                Model('LLM-Research/Molmo-7B-O-0924', 'allenai/Molmo-7B-O-0924'),
+                Model('LLM-Research/Molmo-7B-D-0924', 'allenai/Molmo-7B-D-0924'),
+                Model('LLM-Research/Molmo-72B-0924', 'allenai/Molmo-72B-0924'),
+            ]),
+        ],
+        TemplateType.molmo,
+        get_model_tokenizer_molmo,
+        model_arch=ModelArch.molmo,
+        architectures=['MolmoForCausalLM'],
+        tags=['vision'],
+        requires=['transformers>=4.45'],
+    ))
+
+
+def get_model_tokenizer_megrez_omni(model_dir, *args, **kwargs):
+    model_cls = get_class_from_dynamic_module('modeling_megrezo.MegrezO', model_dir)
+    model_cls._no_split_modules = ['ResidualAttentionBlock', 'LlamaDecoderLayer']
+    model_cls = get_class_from_dynamic_module('modeling_megrezo.SiglipVisionTransformer', model_dir)
+    model_cls._no_split_modules = ['SiglipEncoderLayer']
+    model, processor = get_model_tokenizer_with_flash_attn(model_dir, *args, **kwargs)
+    processor = model._get_or_init_processor()
+    patch_output_clone(model.llm.model.embed_tokens)
+    use_submodel_func(model, 'llm')
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.megrez_omni,
+        [
+            ModelGroup([
+                Model('InfiniAI/Megrez-3B-Omni', 'Infinigence/Megrez-3B-Omni'),
+            ]),
+        ],
+        TemplateType.megrez_omni,
+        get_model_tokenizer_megrez_omni,
+        model_arch=ModelArch.megrez_omni,
+        architectures=['MegrezO'],
+        tags=['vision', 'audio'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_gme, [
+            ModelGroup([
+                Model('iic/gme-Qwen2-VL-2B-Instruct', 'Alibaba-NLP/gme-Qwen2-VL-2B-Instruct'),
+                Model('iic/gme-Qwen2-VL-7B-Instruct', 'Alibaba-NLP/gme-Qwen2-VL-7B-Instruct'),
+            ]),
+        ],
+        TemplateType.qwen2_gme,
+        get_model_tokenizer_qwen2_vl,
+        model_arch=ModelArch.qwen2_vl,
+        architectures=['Qwen2VLForConditionalGeneration'],
+        tags=['vision']))

swift/llm/model/model/moonshot.py

@@ -0,0 +1,47 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from swift.llm import TemplateType
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..patcher import patch_output_clone
+from ..register import (Model, ModelGroup, ModelMeta, get_model_tokenizer_multimodal,
+                        get_model_tokenizer_with_flash_attn, register_model)
+
+register_model(
+    ModelMeta(
+        LLMModelType.moonlight,
+        [
+            ModelGroup([
+                Model('moonshotai/Moonlight-16B-A3B', 'moonshotai/Moonlight-16B-A3B'),
+                Model('moonshotai/Moonlight-16B-A3B-Instruct', 'moonshotai/Moonlight-16B-A3B-Instruct'),
+            ]),
+        ],
+        TemplateType.moonlight,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['DeepseekV3ForCausalLM'],
+        model_arch=ModelArch.deepseek_v2,
+        requires=['transformers<4.49'],
+    ))
+
+
+def get_model_tokenizer_kimi_vl(*args, **kwargs):
+    model, processor = get_model_tokenizer_multimodal(*args, **kwargs)
+    if model is not None:
+        patch_output_clone(model.language_model.model.embed_tokens)
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.kimi_vl,
+        [
+            ModelGroup([
+                Model('moonshotai/Kimi-VL-A3B-Instruct', 'moonshotai/Kimi-VL-A3B-Instruct'),
+                Model('moonshotai/Kimi-VL-A3B-Thinking', 'moonshotai/Kimi-VL-A3B-Thinking'),
+            ])
+        ],
+        TemplateType.kimi_vl,
+        get_model_tokenizer_kimi_vl,
+        architectures=['KimiVLForConditionalGeneration'],
+        model_arch=ModelArch.llava_hf,
+        requires=['transformers<4.49'],
+    ))

swift/llm/model/model/mplug.py

@@ -0,0 +1,142 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import sys
+from functools import partial
+from typing import Any, Dict
+
+from transformers import AutoConfig
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from swift.llm import TemplateType
+from swift.utils import get_logger
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import ModelInfo, git_clone_github, use_submodel_func
+from .qwen import get_model_tokenizer_qwen
+
+logger = get_logger()
+
+
+def get_model_tokenizer_mplug_owl2(model_dir: str,
+                                   model_info: ModelInfo,
+                                   model_kwargs: Dict[str, Any],
+                                   load_model: bool = True,
+                                   **kwargs):
+    local_repo_path = kwargs.get('local_repo_path')
+    if not local_repo_path:
+        local_repo_path = git_clone_github('https://github.com/X-PLUG/mPLUG-Owl')
+    local_repo_path = os.path.join(local_repo_path, 'mPLUG-Owl2')
+    sys.path.append(local_repo_path)
+
+    # register
+    # https://github.com/X-PLUG/mPLUG-Owl/blob/main/mPLUG-Owl2/mplug_owl2/model/modeling_mplug_owl2.py#L447
+    from mplug_owl2 import MPLUGOwl2LlamaForCausalLM
+    from transformers.models.clip.image_processing_clip import CLIPImageProcessor
+    model_config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    vocab_size = kwargs.pop('vocab_size', None)
+    if vocab_size is not None:
+        model_config.vocab_size = vocab_size
+    get_model_tokenizer_function = kwargs.pop('get_model_tokenizer_function', get_model_tokenizer_with_flash_attn)
+    model, tokenizer = get_model_tokenizer_function(
+        model_dir, model_info, model_kwargs, load_model, model_config=model_config, **kwargs)
+    logger.info('Please ignore the unimported warning.')
+    processor = CLIPImageProcessor.from_pretrained(model_dir)
+    processor.tokenizer = tokenizer
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mplug_owl2, [ModelGroup([
+            Model('iic/mPLUG-Owl2', 'MAGAer13/mplug-owl2-llama2-7b'),
+        ])],
+        TemplateType.mplug_owl2,
+        get_model_tokenizer_mplug_owl2,
+        model_arch=ModelArch.mplug_owl2,
+        requires=['transformers<4.35', 'icecream'],
+        tags=['vision']), )
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mplug_owl2_1, [ModelGroup([
+            Model('iic/mPLUG-Owl2.1', 'Mizukiluke/mplug_owl_2_1'),
+        ])],
+        TemplateType.mplug_owl2,
+        partial(
+            get_model_tokenizer_mplug_owl2, vocab_size=151851, get_model_tokenizer_function=get_model_tokenizer_qwen),
+        model_arch=ModelArch.mplug_owl2_1,
+        requires=['transformers<4.35', 'icecream'],
+        tags=['vision']))
+
+
+def get_model_tokenizer_mplug_owl3(model_dir: str,
+                                   model_info: ModelInfo,
+                                   model_kwargs: Dict[str, Any],
+                                   load_model: bool = True,
+                                   **kwargs):
+    get_class_from_dynamic_module('configuration_hyper_qwen2.HyperQwen2Config', model_dir)
+    model_cls = get_class_from_dynamic_module('modeling_mplugowl3.mPLUGOwl3Model', model_dir)
+    model_cls._no_split_modules = ['SiglipEncoderLayer']
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    processor = model.init_processor(tokenizer)
+    if model is not None:
+        func_list = ['generate', 'forward']
+        use_submodel_func(model, 'language_model', func_list)
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mplug_owl3, [
+            ModelGroup([
+                Model('iic/mPLUG-Owl3-1B-241014', 'mPLUG/mPLUG-Owl3-1B-241014'),
+                Model('iic/mPLUG-Owl3-2B-241014', 'mPLUG/mPLUG-Owl3-2B-241014'),
+                Model('iic/mPLUG-Owl3-7B-240728', 'mPLUG/mPLUG-Owl3-7B-240728'),
+            ]),
+        ],
+        TemplateType.mplug_owl3,
+        get_model_tokenizer_mplug_owl3,
+        architectures=['mPLUGOwl3Model'],
+        model_arch=ModelArch.mplug_owl3,
+        requires=['transformers>=4.36', 'icecream', 'decord'],
+        tags=['vision', 'video']))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.mplug_owl3_241101, [
+            ModelGroup([
+                Model('iic/mPLUG-Owl3-7B-241101', 'mPLUG/mPLUG-Owl3-7B-241101'),
+            ]),
+        ],
+        TemplateType.mplug_owl3_241101,
+        get_model_tokenizer_mplug_owl3,
+        architectures=['mPLUGOwl3Model'],
+        model_arch=ModelArch.mplug_owl3,
+        requires=['transformers>=4.36', 'icecream'],
+        tags=['vision', 'video']))
+
+
+def get_model_tokenizer_doc_owl2(model_dir: str,
+                                 model_info: ModelInfo,
+                                 model_kwargs: Dict[str, Any],
+                                 load_model: bool = True,
+                                 **kwargs):
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    processor = model.init_processor(tokenizer, basic_image_size=504, crop_anchors='grid_12')
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.doc_owl2, [
+            ModelGroup([
+                Model('iic/DocOwl2', 'mPLUG/DocOwl2'),
+            ]),
+        ],
+        TemplateType.doc_owl2,
+        get_model_tokenizer_doc_owl2,
+        architectures=['mPLUGDocOwl2'],
+        model_arch=ModelArch.doc_owl2,
+        requires=['transformers>=4.36', 'icecream'],
+        tags=['vision']))

swift/llm/model/model/openbuddy.py

@@ -0,0 +1,85 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from swift.llm import TemplateType
+from swift.utils import get_logger
+from ..constant import LLMModelType
+from ..model_arch import ModelArch
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+
+logger = get_logger()
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_llama,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama-65b-v8-bf16', 'OpenBuddy/openbuddy-llama-65b-v8-bf16'),
+            ]),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama2-13b-v8.1-fp16', 'OpenBuddy/openbuddy-llama2-13b-v8.1-fp16'),
+                Model('OpenBuddy/openbuddy-llama2-70b-v10.1-bf16', 'OpenBuddy/openbuddy-llama2-70b-v10.1-bf16'),
+            ]),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-deepseek-67b-v15.2', 'OpenBuddy/openbuddy-deepseek-67b-v15.2'),
+            ]),
+        ],
+        TemplateType.openbuddy,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['LlamaForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_llama3,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama3-8b-v21.1-8k', 'OpenBuddy/openbuddy-llama3-8b-v21.1-8k'),
+                Model('OpenBuddy/openbuddy-llama3-70b-v21.1-8k', 'OpenBuddy/openbuddy-llama3-70b-v21.1-8k'),
+                Model('OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k', 'OpenBuddy/openbuddy-yi1.5-34b-v21.3-32k'),
+            ]),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k', 'OpenBuddy/openbuddy-llama3.1-8b-v22.1-131k'),
+                Model('OpenBuddy/openbuddy-nemotron-70b-v23.2-131k', 'OpenBuddy/openbuddy-nemotron-70b-v23.2-131k'),
+            ],
+                       requires=['transformers>=4.43']),
+            ModelGroup(
+                [Model('OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k', 'OpenBuddy/openbuddy-llama3.3-70b-v24.3-131k')],
+                requires=['transformers>=4.45'])
+        ],
+        TemplateType.openbuddy2,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['LlamaForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_mistral,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-mistral-7b-v17.1-32k', 'OpenBuddy/openbuddy-mistral-7b-v17.1-32k'),
+            ]),
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-zephyr-7b-v14.1', 'OpenBuddy/openbuddy-zephyr-7b-v14.1'),
+            ]),
+        ],
+        TemplateType.openbuddy,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        requires=['transformers>=4.34'],
+        architectures=['MistralForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.openbuddy_mixtral,
+        [
+            ModelGroup([
+                Model('OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k', 'OpenBuddy/openbuddy-mixtral-7bx8-v18.1-32k'),
+            ], ),
+        ],
+        TemplateType.openbuddy,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['MixtralForCausalLM'],
+        requires=['transformers>=4.36'],
+    ))

swift/llm/model/model/qwen.py

@@ -0,0 +1,899 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, Optional, Tuple, Type
+
+import torch
+from transformers import AutoConfig, BitsAndBytesConfig, PreTrainedTokenizerBase
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+from transformers.models.auto.tokenization_auto import get_tokenizer_config
+
+from swift.llm import TemplateType
+from swift.utils import get_device_count, get_dist_setting, get_env_args, get_logger
+from ..constant import LLMModelType, MLLMModelType, RMModelType
+from ..model_arch import ModelArch
+from ..patcher import patch_fixed_device, patch_output_clone, patch_output_to_input_device
+from ..register import (Model, ModelGroup, ModelMeta, get_model_tokenizer_multimodal, get_model_tokenizer_reward_model,
+                        get_model_tokenizer_with_flash_attn, register_model)
+from ..utils import AttnImpl, ModelInfo, use_submodel_func
+
+logger = get_logger()
+dtype_mapping = {torch.float16: 'fp16', torch.bfloat16: 'bf16', torch.float32: 'fp32'}
+
+
+def get_model_tokenizer_qwen(model_dir: str,
+                             model_info: ModelInfo,
+                             model_kwargs: Dict[str, Any],
+                             load_model: bool = True,
+                             model_config=None,
+                             **kwargs):
+    if model_config is None:
+        model_config = AutoConfig.from_pretrained(model_dir, trust_remote_code=True)
+    if model_info.torch_dtype is not None:
+        k_true = dtype_mapping[model_info.torch_dtype]
+        for k in dtype_mapping.values():
+            setattr(model_config, k, k == k_true)
+
+    quantization_config = model_kwargs.get('quantization_config')
+    if not isinstance(quantization_config, BitsAndBytesConfig):
+        # not bnb quant
+        model_config.torch_dtype = None
+    use_flash_attn = AttnImpl.to_use_flash_attn(kwargs.pop('attn_impl', None), 'auto')
+    model_config.use_flash_attn = use_flash_attn
+    kwargs['model_config'] = model_config
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    try:
+        # fix mp+ddp bug
+        model.transformer.registered_causal_mask = model.transformer.registered_causal_mask.cuda()
+        logger.info('registered_causal_mask to cuda')
+    except AttributeError:
+        pass
+    if tokenizer.eos_token_id is None:
+        tokenizer.eos_token_id = tokenizer.eod_id
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen,
+        [
+            # qwen
+            ModelGroup([
+                # chat
+                Model('Qwen/Qwen-1_8B-Chat', 'Qwen/Qwen-1_8B-Chat'),
+                Model('Qwen/Qwen-7B-Chat', 'Qwen/Qwen-7B-Chat'),
+                Model('Qwen/Qwen-14B-Chat', 'Qwen/Qwen-14B-Chat'),
+                Model('Qwen/Qwen-72B-Chat', 'Qwen/Qwen-72B-Chat'),
+                # base
+                Model('Qwen/Qwen-1_8B', 'Qwen/Qwen-1_8B'),
+                Model('Qwen/Qwen-7B', 'Qwen/Qwen-7B'),
+                Model('Qwen/Qwen-14B', 'Qwen/Qwen-14B'),
+                Model('Qwen/Qwen-72B', 'Qwen/Qwen-72B'),
+                # gptq-int4
+                Model('Qwen/Qwen-1_8B-Chat-Int4', 'Qwen/Qwen-1_8B-Chat-Int4'),
+                Model('Qwen/Qwen-7B-Chat-Int4', 'Qwen/Qwen-7B-Chat-Int4'),
+                Model('Qwen/Qwen-14B-Chat-Int4', 'Qwen/Qwen-14B-Chat-Int4'),
+                Model('Qwen/Qwen-72B-Chat-Int4', 'Qwen/Qwen-72B-Chat-Int4'),
+                # gptq-int8
+                Model('Qwen/Qwen-1_8B-Chat-Int8', 'Qwen/Qwen-1_8B-Chat-Int8'),
+                Model('Qwen/Qwen-7B-Chat-Int8', 'Qwen/Qwen-7B-Chat-Int8'),
+                Model('Qwen/Qwen-14B-Chat-Int8', 'Qwen/Qwen-14B-Chat-Int8'),
+                Model('Qwen/Qwen-72B-Chat-Int8', 'Qwen/Qwen-72B-Chat-Int8'),
+            ]),
+            # tongyi-finance
+            ModelGroup([
+                Model('TongyiFinance/Tongyi-Finance-14B-Chat', 'jxy/Tongyi-Finance-14B-Chat'),
+                Model('TongyiFinance/Tongyi-Finance-14B'),
+                Model('TongyiFinance/Tongyi-Finance-14B-Chat-Int4', 'jxy/Tongyi-Finance-14B-Chat-Int4'),
+            ],
+                       tags=['financial']),
+        ],
+        TemplateType.qwen,
+        get_model_tokenizer_qwen,
+        architectures=['QWenLMHeadModel'],
+        model_arch=ModelArch.qwen))
+
+register_model(
+    ModelMeta(
+        LLMModelType.modelscope_agent,
+        [ModelGroup([
+            Model('iic/ModelScope-Agent-7B'),
+            Model('iic/ModelScope-Agent-14B'),
+        ])],
+        TemplateType.modelscope_agent,
+        get_model_tokenizer_qwen,
+        architectures=['QWenLMHeadModel'],
+        model_arch=ModelArch.qwen))
+
+
+def _qwen_vl_audio_decode(self, *args, skip_special_tokens=False, **kwargs) -> str:
+    if skip_special_tokens:
+        token_ids = kwargs['token_ids']
+        while len(token_ids) > 0 and token_ids[-1] in {151645, 151643}:
+            token_ids.pop()
+        return self._old_decode(*args, skip_special_tokens=False, **kwargs)
+    else:
+        return self._old_decode(*args, skip_special_tokens=False, **kwargs)
+
+
+def fix_qwen_inplace_bug(model) -> None:
+    # qwen-vl, qwen-audio
+    first_drop = model.transformer.drop
+    if first_drop.p == 0.:
+        # fix in-place operation bug
+        patch_output_clone(first_drop)
+
+
+def get_model_tokenizer_qwen_audio(model_dir: str,
+                                   model_info: ModelInfo,
+                                   model_kwargs: Dict[str, Any],
+                                   load_model: bool = True,
+                                   **kwargs):
+    tokenizer_config = get_tokenizer_config(model_dir)
+    class_ref = tokenizer_config['auto_map']['AutoTokenizer'][0]
+    tokenizer_cls: Type[PreTrainedTokenizerBase] = get_class_from_dynamic_module(class_ref, model_dir)
+    tokenizer_cls._auto_class = 'AutoTokenizer'
+    tokenizer_cls.AUDIO_ST = ()  # fix no attr `self.AUDIO_ST` bug
+    if not hasattr(tokenizer_cls, '_old_decode'):
+        tokenizer_cls._old_decode = tokenizer_cls._decode
+        tokenizer_cls._decode = _qwen_vl_audio_decode
+    kwargs['tokenizer'] = tokenizer_cls.from_pretrained(model_dir, trust_remote_code=True)
+    model, tokenizer = get_model_tokenizer_qwen(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if model is not None:
+        fix_qwen_inplace_bug(model)
+
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen_audio, [
+            ModelGroup([
+                Model('Qwen/Qwen-Audio-Chat', 'Qwen/Qwen-Audio-Chat'),
+                Model('Qwen/Qwen-Audio', 'Qwen/Qwen-Audio'),
+            ])
+        ],
+        TemplateType.qwen_audio,
+        get_model_tokenizer_qwen_audio,
+        model_arch=ModelArch.qwen_audio,
+        architectures=['QWenLMHeadModel'],
+        additional_saved_files=['mel_filters.npz'],
+        tags=['audio']))
+
+
+def _qwen_vl_visual_block_forward(
+    self,
+    q_x: torch.Tensor,
+    k_x: Optional[torch.Tensor] = None,
+    v_x: Optional[torch.Tensor] = None,
+    attn_mask: Optional[torch.Tensor] = None,
+):
+    k_x = self.ln_1_kv(k_x) if hasattr(self, 'ln_1_kv') and k_x is not None else None
+    v_x = self.ln_1_kv(v_x) if hasattr(self, 'ln_1_kv') and v_x is not None else None
+
+    x = q_x + self.attention(q_x=self.ln_1(q_x), k_x=k_x, v_x=v_x, attn_mask=attn_mask)
+    z = self.mlp(self.ln_2(x))
+    x = x.to(z.device) + z  # FIX
+    return x
+
+
+def get_model_tokenizer_qwen_vl(model_dir: str,
+                                model_info: ModelInfo,
+                                model_kwargs: Dict[str, Any],
+                                load_model: bool = True,
+                                **kwargs):
+    if (model_kwargs.get('quantization_config') is not None
+            and isinstance(model_kwargs['quantization_config'], BitsAndBytesConfig)):
+        # https://github.com/pytorch/pytorch/issues/58969
+        model_kwargs['quantization_config'].llm_int8_skip_modules = ['lm_head', 'attn_pool.attn']
+        _TransformerBlock = get_class_from_dynamic_module('visual.TransformerBlock', model_dir)
+
+        def _get_cast_dtype(self) -> torch.dtype:
+            return self.resblocks[0].ln_1.weight.dtype
+
+        _TransformerBlock.__old_get_cast_dtype = _TransformerBlock.get_cast_dtype
+        _TransformerBlock.get_cast_dtype = _get_cast_dtype
+
+    tokenizer_config = get_tokenizer_config(model_dir)
+    class_ref = tokenizer_config['auto_map']['AutoTokenizer'][0]
+    tokenizer_cls: Type[PreTrainedTokenizerBase] = get_class_from_dynamic_module(class_ref, model_dir)
+    tokenizer_cls._auto_class = 'AutoTokenizer'
+    tokenizer_cls.IMAGE_ST = ()  # fix no attr `self.IMAGE_ST` bug
+    if not hasattr(tokenizer_cls, '_old_decode'):
+        tokenizer_cls._old_decode = tokenizer_cls._decode
+        tokenizer_cls._decode = _qwen_vl_audio_decode
+    # fix device_map is 4
+    n_gpu = get_device_count()
+    local_world_size = get_dist_setting()[3]
+    if n_gpu // local_world_size >= 4:
+        visual_block_cls = get_class_from_dynamic_module('visual.VisualAttentionBlock', model_dir)
+        visual_block_cls.__old_forward = visual_block_cls.forward
+        visual_block_cls.forward = _qwen_vl_visual_block_forward
+
+    kwargs['tokenizer'] = tokenizer_cls.from_pretrained(model_dir, trust_remote_code=True)
+    model, tokenizer = get_model_tokenizer_qwen(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if model is not None:
+        device_type = next(model.parameters()).device.type
+        fix_qwen_inplace_bug(model)
+        # fix device_map is 4
+        if n_gpu // local_world_size >= 4:
+            model.transformer.visual.proj.data = model.transformer.visual.proj.to(
+                model.transformer.visual.ln_post.bias.device)
+        # fix images cuda:1 bug
+        patch_fixed_device(model.transformer.visual, f'{device_type}:0')
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen_vl, [
+            ModelGroup([
+                Model('Qwen/Qwen-VL-Chat', 'Qwen/Qwen-VL-Chat'),
+                Model('Qwen/Qwen-VL', 'Qwen/Qwen-VL'),
+                Model('Qwen/Qwen-VL-Chat-Int4', 'Qwen/Qwen-VL-Chat-Int4'),
+            ])
+        ],
+        TemplateType.qwen_vl,
+        get_model_tokenizer_qwen_vl,
+        model_arch=ModelArch.qwen_vl,
+        architectures=['QWenLMHeadModel'],
+        additional_saved_files=['SimSun.ttf'],
+        tags=['vision']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen2,
+        [
+            # qwen1.5
+            ModelGroup([
+                # chat
+                Model('Qwen/Qwen1.5-0.5B-Chat', 'Qwen/Qwen1.5-0.5B-Chat'),
+                Model('Qwen/Qwen1.5-1.8B-Chat', 'Qwen/Qwen1.5-1.8B-Chat'),
+                Model('Qwen/Qwen1.5-4B-Chat', 'Qwen/Qwen1.5-4B-Chat'),
+                Model('Qwen/Qwen1.5-7B-Chat', 'Qwen/Qwen1.5-7B-Chat'),
+                Model('Qwen/Qwen1.5-14B-Chat', 'Qwen/Qwen1.5-14B-Chat'),
+                Model('Qwen/Qwen1.5-32B-Chat', 'Qwen/Qwen1.5-32B-Chat'),
+                Model('Qwen/Qwen1.5-72B-Chat', 'Qwen/Qwen1.5-72B-Chat'),
+                Model('Qwen/Qwen1.5-110B-Chat', 'Qwen/Qwen1.5-110B-Chat'),
+                # base
+                Model('Qwen/Qwen1.5-0.5B', 'Qwen/Qwen1.5-0.5B'),
+                Model('Qwen/Qwen1.5-1.8B', 'Qwen/Qwen1.5-1.8B'),
+                Model('Qwen/Qwen1.5-4B', 'Qwen/Qwen1.5-4B'),
+                Model('Qwen/Qwen1.5-7B', 'Qwen/Qwen1.5-7B'),
+                Model('Qwen/Qwen1.5-14B', 'Qwen/Qwen1.5-14B'),
+                Model('Qwen/Qwen1.5-32B', 'Qwen/Qwen1.5-32B'),
+                Model('Qwen/Qwen1.5-72B', 'Qwen/Qwen1.5-72B'),
+                Model('Qwen/Qwen1.5-110B', 'Qwen/Qwen1.5-110B'),
+                # gptq-int4
+                Model('Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-4B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-4B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-7B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-7B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-14B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-14B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-32B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-32B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-72B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-72B-Chat-GPTQ-Int4'),
+                Model('Qwen/Qwen1.5-110B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-110B-Chat-GPTQ-Int4'),
+                # gptq-int8
+                Model('Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-0.5B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-1.8B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-4B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-4B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-7B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-7B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-14B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-14B-Chat-GPTQ-Int8'),
+                Model('Qwen/Qwen1.5-72B-Chat-GPTQ-Int8', 'Qwen/Qwen1.5-72B-Chat-GPTQ-Int8'),
+                # awq-int4
+                Model('Qwen/Qwen1.5-0.5B-Chat-AWQ', 'Qwen/Qwen1.5-0.5B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-1.8B-Chat-AWQ', 'Qwen/Qwen1.5-1.8B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-4B-Chat-AWQ', 'Qwen/Qwen1.5-4B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-7B-Chat-AWQ', 'Qwen/Qwen1.5-7B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-14B-Chat-AWQ', 'Qwen/Qwen1.5-14B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-32B-Chat-AWQ', 'Qwen/Qwen1.5-32B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-72B-Chat-AWQ', 'Qwen/Qwen1.5-72B-Chat-AWQ'),
+                Model('Qwen/Qwen1.5-110B-Chat-AWQ', 'Qwen/Qwen1.5-110B-Chat-AWQ'),
+            ]),
+            # code-qwen1.5
+            ModelGroup([
+                Model('Qwen/CodeQwen1.5-7B', 'Qwen/CodeQwen1.5-7B'),
+                Model('Qwen/CodeQwen1.5-7B-Chat', 'Qwen/CodeQwen1.5-7B-Chat'),
+                Model('Qwen/CodeQwen1.5-7B-Chat-AWQ', 'Qwen/CodeQwen1.5-7B-Chat-AWQ'),
+            ],
+                       tags=['coding']),
+            # qwen2
+            ModelGroup([
+                # instruct
+                Model('Qwen/Qwen2-0.5B-Instruct', 'Qwen/Qwen2-0.5B-Instruct'),
+                Model('Qwen/Qwen2-1.5B-Instruct', 'Qwen/Qwen2-1.5B-Instruct'),
+                Model('Qwen/Qwen2-7B-Instruct', 'Qwen/Qwen2-7B-Instruct'),
+                Model('Qwen/Qwen2-72B-Instruct', 'Qwen/Qwen2-72B-Instruct'),
+                # base
+                Model('Qwen/Qwen2-0.5B', 'Qwen/Qwen2-0.5B'),
+                Model('Qwen/Qwen2-1.5B', 'Qwen/Qwen2-1.5B'),
+                Model('Qwen/Qwen2-7B', 'Qwen/Qwen2-7B'),
+                Model('Qwen/Qwen2-72B', 'Qwen/Qwen2-72B'),
+                # gptq-int4
+                Model('Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-0.5B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-1.5B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2-7B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-7B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2-72B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-72B-Instruct-GPTQ-Int4'),
+                # gptq-int8
+                Model('Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-0.5B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-1.5B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2-7B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-7B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2-72B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-72B-Instruct-GPTQ-Int8'),
+                # awq-int4
+                Model('Qwen/Qwen2-0.5B-Instruct-AWQ', 'Qwen/Qwen2-0.5B-Instruct-AWQ'),
+                Model('Qwen/Qwen2-1.5B-Instruct-AWQ', 'Qwen/Qwen2-1.5B-Instruct-AWQ'),
+                Model('Qwen/Qwen2-7B-Instruct-AWQ', 'Qwen/Qwen2-7B-Instruct-AWQ'),
+                Model('Qwen/Qwen2-72B-Instruct-AWQ', 'Qwen/Qwen2-72B-Instruct-AWQ'),
+            ]),
+            # qwen2-math
+            ModelGroup(
+                [
+                    # instruct
+                    Model('Qwen/Qwen2-Math-1.5B-Instruct', 'Qwen/Qwen2-Math-1.5B-Instruct'),
+                    Model('Qwen/Qwen2-Math-7B-Instruct', 'Qwen/Qwen2-Math-7B-Instruct'),
+                    Model('Qwen/Qwen2-Math-72B-Instruct', 'Qwen/Qwen2-Math-72B-Instruct'),
+                    # base
+                    Model('Qwen/Qwen2-Math-1.5B', 'Qwen/Qwen2-Math-1.5B'),
+                    Model('Qwen/Qwen2-Math-7B', 'Qwen/Qwen2-Math-7B'),
+                    Model('Qwen/Qwen2-Math-72B', 'Qwen/Qwen2-Math-72B'),
+                ],
+                tags=['math']),
+            # qwen2.5-1m
+            ModelGroup([
+                Model('Qwen/Qwen2.5-7B-Instruct-1M', 'Qwen/Qwen2.5-7B-Instruct-1M'),
+                Model('Qwen/Qwen2.5-14B-Instruct-1M', 'Qwen/Qwen2.5-14B-Instruct-1M'),
+            ]),
+            # other
+            ModelGroup([Model('PowerInfer/SmallThinker-3B-Preview', 'PowerInfer/SmallThinker-3B-Preview')]),
+        ],
+        TemplateType.qwen,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen2_5,
+        [
+            # qwen2.5
+            ModelGroup([
+                # instruct
+                Model('Qwen/Qwen2.5-0.5B-Instruct', 'Qwen/Qwen2.5-0.5B-Instruct'),
+                Model('Qwen/Qwen2.5-1.5B-Instruct', 'Qwen/Qwen2.5-1.5B-Instruct'),
+                Model('Qwen/Qwen2.5-3B-Instruct', 'Qwen/Qwen2.5-3B-Instruct'),
+                Model('Qwen/Qwen2.5-7B-Instruct', 'Qwen/Qwen2.5-7B-Instruct'),
+                Model('Qwen/Qwen2.5-14B-Instruct', 'Qwen/Qwen2.5-14B-Instruct'),
+                Model('Qwen/Qwen2.5-32B-Instruct', 'Qwen/Qwen2.5-32B-Instruct'),
+                Model('Qwen/Qwen2.5-72B-Instruct', 'Qwen/Qwen2.5-72B-Instruct'),
+                # base
+                Model('Qwen/Qwen2.5-0.5B', 'Qwen/Qwen2.5-0.5B'),
+                Model('Qwen/Qwen2.5-1.5B', 'Qwen/Qwen2.5-1.5B'),
+                Model('Qwen/Qwen2.5-3B', 'Qwen/Qwen2.5-3B'),
+                Model('Qwen/Qwen2.5-7B', 'Qwen/Qwen2.5-7B'),
+                Model('Qwen/Qwen2.5-14B', 'Qwen/Qwen2.5-14B'),
+                Model('Qwen/Qwen2.5-32B', 'Qwen/Qwen2.5-32B'),
+                Model('Qwen/Qwen2.5-72B', 'Qwen/Qwen2.5-72B'),
+                # gptq-int4
+                Model('Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-3B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-7B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-14B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-32B-Instruct-GPTQ-Int4'),
+                Model('Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-72B-Instruct-GPTQ-Int4'),
+                # gptq-int8
+                Model('Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-0.5B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-1.5B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-3B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-7B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-14B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-32B-Instruct-GPTQ-Int8'),
+                Model('Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-72B-Instruct-GPTQ-Int8'),
+                # awq-int4
+                Model('Qwen/Qwen2.5-0.5B-Instruct-AWQ', 'Qwen/Qwen2.5-0.5B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-1.5B-Instruct-AWQ', 'Qwen/Qwen2.5-1.5B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-3B-Instruct-AWQ', 'Qwen/Qwen2.5-3B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-7B-Instruct-AWQ', 'Qwen/Qwen2.5-7B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-14B-Instruct-AWQ', 'Qwen/Qwen2.5-14B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-32B-Instruct-AWQ', 'Qwen/Qwen2.5-32B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-72B-Instruct-AWQ', 'Qwen/Qwen2.5-72B-Instruct-AWQ'),
+            ]),
+            # qwen2.5-coder
+            ModelGroup(
+                [
+                    # instruct
+                    Model('Qwen/Qwen2.5-Coder-0.5B-Instruct', 'Qwen/Qwen2.5-Coder-0.5B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B-Instruct', 'Qwen/Qwen2.5-Coder-1.5B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-3B-Instruct', 'Qwen/Qwen2.5-Coder-3B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-7B-Instruct', 'Qwen/Qwen2.5-Coder-7B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-14B-Instruct', 'Qwen/Qwen2.5-Coder-14B-Instruct'),
+                    Model('Qwen/Qwen2.5-Coder-32B-Instruct', 'Qwen/Qwen2.5-Coder-32B-Instruct'),
+                    # base
+                    Model('Qwen/Qwen2.5-Coder-0.5B', 'Qwen/Qwen2.5-Coder-0.5B'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B', 'Qwen/Qwen2.5-Coder-1.5B'),
+                    Model('Qwen/Qwen2.5-Coder-3B', 'Qwen/Qwen2.5-Coder-3B'),
+                    Model('Qwen/Qwen2.5-Coder-7B', 'Qwen/Qwen2.5-Coder-7B'),
+                    Model('Qwen/Qwen2.5-Coder-14B', 'Qwen/Qwen2.5-Coder-14B'),
+                    Model('Qwen/Qwen2.5-Coder-32B', 'Qwen/Qwen2.5-Coder-32B'),
+                    # AWQ
+                    Model('Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-0.5B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-1.5B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-3B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-3B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-7B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-7B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-14B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-14B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2.5-Coder-32B-Instruct-AWQ', 'Qwen/Qwen2.5-Coder-32B-Instruct-AWQ'),
+                    # GPTQ
+                    Model('Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-0.5B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-1.5B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-3B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-7B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-14B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4', 'Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8', 'Qwen/Qwen2.5-Coder-32B-Instruct-GPTQ-Int8'),
+                ],
+                tags=['coding'])
+        ],
+        TemplateType.qwen2_5,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen2_5_math,
+        [
+            # qwen2.5-math
+            ModelGroup(
+                [
+                    # instruct
+                    Model('Qwen/Qwen2.5-Math-1.5B-Instruct', 'Qwen/Qwen2.5-Math-1.5B-Instruct'),
+                    Model('Qwen/Qwen2.5-Math-7B-Instruct', 'Qwen/Qwen2.5-Math-7B-Instruct'),
+                    Model('Qwen/Qwen2.5-Math-72B-Instruct', 'Qwen/Qwen2.5-Math-72B-Instruct'),
+                    # base
+                    Model('Qwen/Qwen2.5-Math-1.5B', 'Qwen/Qwen2.5-Math-1.5B'),
+                    Model('Qwen/Qwen2.5-Math-7B', 'Qwen/Qwen2.5-Math-7B'),
+                    Model('Qwen/Qwen2.5-Math-72B', 'Qwen/Qwen2.5-Math-72B'),
+                ],
+                tags=['math']),
+        ],
+        TemplateType.qwen2_5_math,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen2_moe,
+        [
+            # qwen1.5-moe
+            ModelGroup([
+                Model('Qwen/Qwen1.5-MoE-A2.7B-Chat', 'Qwen/Qwen1.5-MoE-A2.7B-Chat'),
+                Model('Qwen/Qwen1.5-MoE-A2.7B', 'Qwen/Qwen1.5-MoE-A2.7B'),
+                Model('Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4', 'Qwen/Qwen1.5-MoE-A2.7B-Chat-GPTQ-Int4'),
+            ]),
+            ModelGroup([
+                Model('Qwen/Qwen2-57B-A14B-Instruct', 'Qwen/Qwen2-57B-A14B-Instruct'),
+                Model('Qwen/Qwen2-57B-A14B', 'Qwen/Qwen2-57B-A14B'),
+                Model('Qwen/Qwen2-57B-A14B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-57B-A14B-Instruct-GPTQ-Int4'),
+            ])
+        ],
+        TemplateType.qwen,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen2MoeForCausalLM'],
+        requires=['transformers>=4.40'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen3,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen3-0.6B-Base', 'Qwen/Qwen3-0.6B-Base'),
+                Model('Qwen/Qwen3-1.7B-Base', 'Qwen/Qwen3-1.7B-Base'),
+                Model('Qwen/Qwen3-4B-Base', 'Qwen/Qwen3-4B-Base'),
+                Model('Qwen/Qwen3-8B-Base', 'Qwen/Qwen3-8B-Base'),
+                Model('Qwen/Qwen3-14B-Base', 'Qwen/Qwen3-14B-Base'),
+                Model('Qwen/Qwen3-32B-Base', 'Qwen/Qwen3-32B-Base'),
+                # instruct
+                Model('Qwen/Qwen3-0.6B', 'Qwen/Qwen3-0.6B'),
+                Model('Qwen/Qwen3-1.7B', 'Qwen/Qwen3-1.7B'),
+                Model('Qwen/Qwen3-4B', 'Qwen/Qwen3-4B'),
+                Model('Qwen/Qwen3-8B', 'Qwen/Qwen3-8B'),
+                Model('Qwen/Qwen3-14B', 'Qwen/Qwen3-14B'),
+                Model('Qwen/Qwen3-32B', 'Qwen/Qwen3-32B'),
+                # fp8
+                Model('Qwen/Qwen3-0.6B-FP8', 'Qwen/Qwen3-0.6B-FP8'),
+                Model('Qwen/Qwen3-1.7B-FP8', 'Qwen/Qwen3-1.7B-FP8'),
+                Model('Qwen/Qwen3-4B-FP8', 'Qwen/Qwen3-4B-FP8'),
+                Model('Qwen/Qwen3-8B-FP8', 'Qwen/Qwen3-8B-FP8'),
+                Model('Qwen/Qwen3-14B-FP8', 'Qwen/Qwen3-14B-FP8'),
+                Model('Qwen/Qwen3-32B-FP8', 'Qwen/Qwen3-32B-FP8'),
+                # awq
+                Model('Qwen/Qwen3-4B-AWQ', 'Qwen/Qwen3-4B-AWQ'),
+                Model('Qwen/Qwen3-8B-AWQ', 'Qwen/Qwen3-8B-AWQ'),
+                Model('Qwen/Qwen3-14B-AWQ', 'Qwen/Qwen3-14B-AWQ'),
+                Model('Qwen/Qwen3-32B-AWQ', 'Qwen/Qwen3-32B-AWQ'),
+                # swift
+                Model('swift/Qwen3-32B-AWQ'),
+            ]),
+        ],
+        TemplateType.qwen3,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen3ForCausalLM'],
+        requires=['transformers>=4.51'],
+        model_arch=ModelArch.llama))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwen3_moe,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen3-30B-A3B-Base', 'Qwen/Qwen3-30B-A3B-Base'),
+                Model('Qwen/Qwen3-235B-A22B-Base', 'Qwen/Qwen3-235B-A22B-Base'),
+                # instruct
+                Model('Qwen/Qwen3-30B-A3B', 'Qwen/Qwen3-30B-A3B'),
+                Model('Qwen/Qwen3-235B-A22B', 'Qwen/Qwen3-235B-A22B'),
+                # fp8
+                Model('Qwen/Qwen3-30B-A3B-FP8', 'Qwen/Qwen3-30B-A3B-FP8'),
+                Model('Qwen/Qwen3-235B-A22B-FP8', 'Qwen/Qwen3-235B-A22B-FP8'),
+                # awq
+                Model('swift/Qwen3-30B-A3B-AWQ', 'cognitivecomputations/Qwen3-30B-A3B-AWQ'),
+                Model('swift/Qwen3-235B-A22B-AWQ', 'cognitivecomputations/Qwen3-235B-A22B-AWQ'),
+            ]),
+        ],
+        TemplateType.qwen3,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['Qwen3MoeForCausalLM'],
+        requires=['transformers>=4.51'],
+    ))
+
+
+def patch_qwen_vl_utils(vision_process):
+    if hasattr(vision_process, '_patch'):
+        return
+    for key in [
+            'image_factor', 'min_pixels', 'max_pixels', 'max_ratio', 'video_min_pixels', 'video_max_pixels',
+            'video_total_pixels', 'frame_factor', 'fps', 'fps_min_frames', 'fps_max_frames'
+    ]:
+        type_func = float if key == 'fps' else int
+        setattr(vision_process, key.upper(), get_env_args(key, type_func, getattr(vision_process, key.upper())))
+    _read_video_decord = vision_process._read_video_decord
+
+    def _new_read_video_decord(ele: dict):
+        from swift.llm import load_file
+        ele['video'] = load_file(ele['video'])
+        return _read_video_decord(ele)
+
+    vision_process.VIDEO_READER_BACKENDS['decord'] = _new_read_video_decord
+    vision_process._patch = True
+
+
+def get_model_tokenizer_qwen2_vl(*args, **kwargs):
+    from transformers import Qwen2VLForConditionalGeneration
+    kwargs['automodel_class'] = kwargs['automodel_class'] or Qwen2VLForConditionalGeneration
+    model, tokenizer = get_model_tokenizer_multimodal(*args, **kwargs)
+    if model is not None and hasattr(model.model, 'embed_tokens'):
+        patch_output_clone(model.model.embed_tokens)
+        patch_output_to_input_device(model.model.embed_tokens)
+
+    from qwen_vl_utils import vision_process
+    patch_qwen_vl_utils(vision_process)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_vl,
+        [
+            ModelGroup(
+                [
+                    # chat
+                    Model('Qwen/Qwen2-VL-2B-Instruct', 'Qwen/Qwen2-VL-2B-Instruct'),
+                    Model('Qwen/Qwen2-VL-7B-Instruct', 'Qwen/Qwen2-VL-7B-Instruct'),
+                    Model('Qwen/Qwen2-VL-72B-Instruct', 'Qwen/Qwen2-VL-72B-Instruct'),
+                    # base
+                    Model('Qwen/Qwen2-VL-2B', 'Qwen/Qwen2-VL-2B'),
+                    Model('Qwen/Qwen2-VL-7B', 'Qwen/Qwen2-VL-7B'),
+                    Model('Qwen/Qwen2-VL-72B', 'Qwen/Qwen2-VL-72B'),
+                    # gptq-int4
+                    Model('Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int4'),
+                    Model('Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4', 'Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int4'),
+                    # gptq-int8
+                    Model('Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-VL-7B-Instruct-GPTQ-Int8'),
+                    Model('Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int8', 'Qwen/Qwen2-VL-72B-Instruct-GPTQ-Int8'),
+                    # awq-int4
+                    Model('Qwen/Qwen2-VL-2B-Instruct-AWQ', 'Qwen/Qwen2-VL-2B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2-VL-7B-Instruct-AWQ', 'Qwen/Qwen2-VL-7B-Instruct-AWQ'),
+                    Model('Qwen/Qwen2-VL-72B-Instruct-AWQ', 'Qwen/Qwen2-VL-72B-Instruct-AWQ'),
+                ], ),
+            ModelGroup([
+                Model('bytedance-research/UI-TARS-2B-SFT', 'bytedance-research/UI-TARS-2B-SFT'),
+                Model('bytedance-research/UI-TARS-7B-SFT', 'bytedance-research/UI-TARS-7B-SFT'),
+                Model('bytedance-research/UI-TARS-7B-DPO', 'bytedance-research/UI-TARS-7B-DPO'),
+                Model('bytedance-research/UI-TARS-72B-SFT', 'bytedance-research/UI-TARS-72B-SFT'),
+                Model('bytedance-research/UI-TARS-72B-DPO', 'bytedance-research/UI-TARS-72B-DPO'),
+            ]),
+            ModelGroup([
+                Model('allenai/olmOCR-7B-0225-preview', 'allenai/olmOCR-7B-0225-preview'),
+            ]),
+        ],
+        TemplateType.qwen2_vl,
+        get_model_tokenizer_qwen2_vl,
+        model_arch=ModelArch.qwen2_vl,
+        architectures=['Qwen2VLForConditionalGeneration'],
+        requires=['transformers>=4.45', 'qwen_vl_utils>=0.0.6', 'decord'],
+        tags=['vision', 'video']))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qvq, [
+            ModelGroup([
+                Model('Qwen/QVQ-72B-Preview', 'Qwen/QVQ-72B-Preview'),
+            ]),
+        ],
+        TemplateType.qvq,
+        get_model_tokenizer_qwen2_vl,
+        model_arch=ModelArch.qwen2_vl,
+        architectures=['Qwen2VLForConditionalGeneration'],
+        requires=['transformers>=4.45', 'qwen_vl_utils>=0.0.6', 'decord'],
+        tags=['vision', 'video']))
+
+
+def get_model_tokenizer_qwen2_5_vl(*args, **kwargs):
+    from transformers import Qwen2_5_VLForConditionalGeneration
+    kwargs['automodel_class'] = kwargs['automodel_class'] or Qwen2_5_VLForConditionalGeneration
+    return get_model_tokenizer_qwen2_vl(*args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_5_vl, [
+            ModelGroup([
+                Model('Qwen/Qwen2.5-VL-3B-Instruct', 'Qwen/Qwen2.5-VL-3B-Instruct'),
+                Model('Qwen/Qwen2.5-VL-7B-Instruct', 'Qwen/Qwen2.5-VL-7B-Instruct'),
+                Model('Qwen/Qwen2.5-VL-32B-Instruct', 'Qwen/Qwen2.5-VL-32B-Instruct'),
+                Model('Qwen/Qwen2.5-VL-72B-Instruct', 'Qwen/Qwen2.5-VL-72B-Instruct'),
+            ]),
+            ModelGroup([
+                Model('Qwen/Qwen2.5-VL-3B-Instruct-AWQ', 'Qwen/Qwen2.5-VL-3B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-VL-7B-Instruct-AWQ', 'Qwen/Qwen2.5-VL-7B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-VL-32B-Instruct-AWQ', 'Qwen/Qwen2.5-VL-32B-Instruct-AWQ'),
+                Model('Qwen/Qwen2.5-VL-72B-Instruct-AWQ', 'Qwen/Qwen2.5-VL-72B-Instruct-AWQ'),
+            ]),
+        ],
+        TemplateType.qwen2_5_vl,
+        get_model_tokenizer_qwen2_5_vl,
+        model_arch=ModelArch.qwen2_vl,
+        architectures=['Qwen2_5_VLForConditionalGeneration'],
+        requires=['transformers>=4.49', 'qwen_vl_utils>=0.0.6', 'decord'],
+        tags=['vision', 'video']))
+
+
+def get_model_tokenizer_qwen2_5_omni(model_dir, *args, **kwargs):
+    from transformers import Qwen2_5OmniForConditionalGeneration, Qwen2_5OmniProcessor, Qwen2_5OmniConfig
+    from qwen_omni_utils import vision_process
+    kwargs['automodel_class'] = kwargs['automodel_class'] or Qwen2_5OmniForConditionalGeneration
+    processor = Qwen2_5OmniProcessor.from_pretrained(model_dir, trust_remote_code=True)
+    kwargs['tokenizer'] = processor.tokenizer
+    kwargs['model_config'] = Qwen2_5OmniConfig.from_pretrained(model_dir, trust_remote_code=True)
+    patch_qwen_vl_utils(vision_process)
+    kwargs['model_config'].enable_audio_output = get_env_args('ENABLE_AUDIO_OUTPUT', bool, True)
+    model, _ = get_model_tokenizer_with_flash_attn(model_dir, *args, **kwargs)
+    if model:
+        use_submodel_func(model, 'thinker')
+        model.config.keys_to_ignore_at_inference += ['hidden_states', 'attention_mask']
+        model.config.talker_config.pad_token_id = None
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_5_omni,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2.5-Omni-3B', 'Qwen/Qwen2.5-Omni-3B'),
+                Model('Qwen/Qwen2.5-Omni-7B', 'Qwen/Qwen2.5-Omni-7B'),
+            ]),
+        ],
+        TemplateType.qwen2_5_omni,
+        get_model_tokenizer_qwen2_5_omni,
+        model_arch=ModelArch.qwen2_5_omni,
+        architectures=['Qwen2_5OmniModel'],
+        requires=['transformers>=4.50', 'soundfile', 'qwen_omni_utils', 'decord'],
+        tags=['vision', 'video', 'audio'],
+        additional_saved_files=['spk_dict.pt'],
+        ignore_patterns=[],
+    ))
+
+
+def get_model_tokenizer_qwen2_audio(*args, **kwargs):
+    from transformers import Qwen2AudioForConditionalGeneration
+    kwargs['automodel_class'] = kwargs['automodel_class'] or Qwen2AudioForConditionalGeneration
+    return get_model_tokenizer_multimodal(*args, **kwargs)
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.qwen2_audio,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2-Audio-7B-Instruct', 'Qwen/Qwen2-Audio-7B-Instruct'),
+                Model('Qwen/Qwen2-Audio-7B', 'Qwen/Qwen2-Audio-7B'),
+            ]),
+        ],
+        TemplateType.qwen2_audio,
+        get_model_tokenizer_qwen2_audio,
+        model_arch=ModelArch.qwen2_audio,
+        architectures=['Qwen2AudioForConditionalGeneration'],
+        requires=['transformers>=4.45,<4.49', 'librosa'],
+        tags=['audio'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.marco_o1, [ModelGroup([Model('AIDC-AI/Marco-o1', 'AIDC-AI/Marco-o1')])],
+        TemplateType.marco_o1,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwq_preview, [ModelGroup([Model('Qwen/QwQ-32B-Preview', 'Qwen/QwQ-32B-Preview')])],
+        TemplateType.qwq_preview,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.qwq,
+        [ModelGroup([
+            Model('Qwen/QwQ-32B', 'Qwen/QwQ-32B'),
+            Model('Qwen/QwQ-32B-AWQ', 'Qwen/QwQ-32B-AWQ'),
+        ])],
+        TemplateType.qwq,
+        get_model_tokenizer_with_flash_attn,
+        model_arch=ModelArch.llama,
+        architectures=['Qwen2ForCausalLM'],
+        requires=['transformers>=4.37']))
+
+
+def get_model_tokenizer_ovis(*args, **kwargs):
+    kwargs['attn_impl_keys'] = ['llm_attn_implementation']
+    model, tokenizer = get_model_tokenizer_with_flash_attn(*args, **kwargs)
+    if model is not None:
+        model.visual_tokenizer.to(model.dtype)
+        model.vte.to(model.dtype)
+
+        model.generation_config.cache_implementation = None
+        func_list = ['generate', 'forward', 'get_input_embeddings']
+        use_submodel_func(model, 'llm', func_list)
+        embedding = model.get_input_embeddings()
+        patch_output_clone(embedding)
+    try:
+        # fix device_map
+        from transformers.cache_utils import HybridCache
+
+        def update(self, key_states: torch.Tensor, value_states: torch.Tensor, layer_idx: int, *args,
+                   **kwargs) -> Tuple[torch.Tensor]:
+            self.key_cache[layer_idx] = self.key_cache[layer_idx].to(key_states.device)
+            self.value_cache[layer_idx] = self.value_cache[layer_idx].to(value_states.device)
+            return self._update_origin(key_states, value_states, layer_idx, *args, **kwargs)
+
+        if not hasattr(HybridCache, '_update_origin'):
+            HybridCache._update_origin = HybridCache.update
+            HybridCache.update = update
+    except ImportError:
+        pass
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.ovis1_6,
+        [
+            ModelGroup([
+                Model('AIDC-AI/Ovis1.6-Gemma2-9B', 'AIDC-AI/Ovis1.6-Gemma2-9B'),
+                Model('AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4', 'AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4'),
+                Model('AIDC-AI/Ovis1.6-Gemma2-27B', 'AIDC-AI/Ovis1.6-Gemma2-27B'),
+            ]),
+        ],
+        TemplateType.ovis1_6,
+        get_model_tokenizer_ovis,
+        model_arch=ModelArch.ovis1_6,
+        architectures=['Ovis'],
+        tags=['vision'],
+        requires=['transformers>=4.42'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.ovis1_6_llama3,
+        [
+            ModelGroup([
+                Model('AIDC-AI/Ovis1.6-Llama3.2-3B', 'AIDC-AI/Ovis1.6-Llama3.2-3B'),
+            ]),
+        ],
+        TemplateType.ovis1_6_llama3,
+        get_model_tokenizer_ovis,
+        model_arch=ModelArch.ovis1_6,
+        architectures=['Ovis'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        MLLMModelType.ovis2,
+        [
+            ModelGroup([
+                Model('AIDC-AI/Ovis2-1B', 'AIDC-AI/Ovis2-1B'),
+                Model('AIDC-AI/Ovis2-2B', 'AIDC-AI/Ovis2-2B'),
+                Model('AIDC-AI/Ovis2-4B', 'AIDC-AI/Ovis2-4B'),
+                Model('AIDC-AI/Ovis2-8B', 'AIDC-AI/Ovis2-8B'),
+                Model('AIDC-AI/Ovis2-16B', 'AIDC-AI/Ovis2-16B'),
+                Model('AIDC-AI/Ovis2-34B', 'AIDC-AI/Ovis2-34B'),
+            ]),
+        ],
+        TemplateType.ovis2,
+        get_model_tokenizer_ovis,
+        model_arch=ModelArch.ovis1_6,
+        architectures=['Ovis'],
+        tags=['vision'],
+        requires=['transformers>=4.46.2', 'moviepy<2'],
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.qwen2_reward,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2-Math-RM-72B', 'Qwen/Qwen2-Math-RM-72B'),
+            ]),
+        ],
+        TemplateType.qwen,
+        get_model_tokenizer_reward_model,
+        architectures=['Qwen2ForRewardModel'],
+        requires=['transformers>=4.37'],
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.qwen2_5_prm,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2.5-Math-PRM-7B', 'Qwen/Qwen2.5-Math-PRM-7B'),
+                Model('Qwen/Qwen2.5-Math-7B-PRM800K', 'Qwen/Qwen2.5-Math-7B-PRM800K'),
+                Model('Qwen/Qwen2.5-Math-PRM-72B', 'Qwen/Qwen2.5-Math-PRM-72B'),
+            ]),
+        ],
+        TemplateType.qwen2_5_math_prm,
+        get_model_tokenizer_reward_model,
+        task_type='prm',
+        architectures=['Qwen2ForProcessRewardModel'],
+        requires=['transformers>=4.37'],
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.qwen2_5_math_reward,
+        [
+            ModelGroup([
+                Model('Qwen/Qwen2.5-Math-RM-72B', 'Qwen/Qwen2.5-Math-RM-72B'),
+            ]),
+        ],
+        TemplateType.qwen2_5_math,
+        get_model_tokenizer_reward_model,
+        architectures=['Qwen2ForRewardModel'],
+        requires=['transformers>=4.37'],
+    ))

swift/llm/model/model/skywork.py

@@ -0,0 +1,89 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict
+
+from swift.llm import TemplateType
+from ..constant import LLMModelType, RMModelType
+from ..model_arch import ModelArch
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import ModelInfo
+
+
+def get_skywork_model_tokenizer(model_dir: str,
+                                model_info: ModelInfo,
+                                model_kwargs: Dict[str, Any],
+                                load_model: bool = True,
+                                **kwargs):
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if 'chat' in model_dir:
+        tokenizer.add_tokens('[USER]')
+        tokenizer.add_tokens('[BOT]')
+        tokenizer.add_tokens('[SEP]')
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.skywork,
+        [
+            ModelGroup([
+                Model('skywork/Skywork-13B-base', 'skywork/Skywork-13B-base'),
+                Model('skywork/Skywork-13B-chat'),
+            ]),
+        ],
+        TemplateType.skywork,
+        get_skywork_model_tokenizer,
+        architectures=['SkyworkForCausalLM'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.skywork_o1,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Skywork-o1-Open-Llama-3.1-8B', 'Skywork/Skywork-o1-Open-Llama-3.1-8B'),
+            ]),
+        ],
+        TemplateType.skywork_o1,
+        get_model_tokenizer_with_flash_attn,
+        architectures=['LlamaForCausalLM'],
+        requires=['transformers>=4.43'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.llama3_2_reward,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Skywork-Reward-Llama-3.1-8B', 'Skywork/Skywork-Reward-Llama-3.1-8B'),
+                Model('AI-ModelScope/Skywork-Reward-Llama-3.1-8B-v0.2', 'Skywork/Skywork-Reward-Llama-3.1-8B-v0.2'),
+            ]),
+            ModelGroup([
+                Model('AI-ModelScope/GRM_Llama3.1_8B_rewardmodel-ft', 'Ray2333/GRM_Llama3.1_8B_rewardmodel-ft'),
+                Model('AI-ModelScope/GRM-llama3.2-3B-rewardmodel-ft', 'Ray2333/GRM-llama3.2-3B-rewardmodel-ft'),
+            ])
+        ],
+        TemplateType.llama3_2,
+        get_model_tokenizer_with_flash_attn,
+        requires=['transformers>=4.43'],
+        architectures=['LlamaForSequenceClassification'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        RMModelType.gemma_reward,
+        [
+            ModelGroup([
+                Model('AI-ModelScope/Skywork-Reward-Gemma-2-27B', 'Skywork/Skywork-Reward-Gemma-2-27B'),
+                Model('AI-ModelScope/Skywork-Reward-Gemma-2-27B-v0.2', 'Skywork/Skywork-Reward-Gemma-2-27B-v0.2'),
+            ]),
+        ],
+        TemplateType.gemma,
+        get_model_tokenizer_with_flash_attn,
+        requires=['transformers>=4.42'],
+        architectures=['Gemma2ForSequenceClassification'],
+        model_arch=ModelArch.llama,
+    ))

swift/llm/model/model/stepfun.py

@@ -0,0 +1,86 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import shutil
+import sys
+
+from transformers import AutoModel
+
+from swift.llm import TemplateType
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..register import (Model, ModelGroup, ModelMeta, get_model_tokenizer_multimodal,
+                        get_model_tokenizer_with_flash_attn, register_model)
+from ..utils import git_clone_github, safe_snapshot_download
+
+
+def get_model_tokenizer_got_ocr2(*args, **kwargs):
+    kwargs['automodel_class'] = AutoModel
+    model, tokenizer = get_model_tokenizer_with_flash_attn(*args, **kwargs)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.got_ocr2, [
+            ModelGroup([
+                Model('stepfun-ai/GOT-OCR2_0', 'stepfun-ai/GOT-OCR2_0'),
+            ]),
+        ],
+        TemplateType.got_ocr2,
+        get_model_tokenizer_got_ocr2,
+        model_arch=ModelArch.got_ocr2,
+        architectures=['GOTQwenForCausalLM'],
+        tags=['vision']))
+
+
+def get_model_tokenizer_got_ocr2_hf(model_dir, *args, **kwargs):
+    from transformers.models.got_ocr2 import GotOcr2ForConditionalGeneration
+    GotOcr2ForConditionalGeneration._no_split_modules.append('GotOcr2VisionLayer')
+    model, processor = get_model_tokenizer_multimodal(model_dir, *args, **kwargs)
+    return model, processor
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.got_ocr2_hf, [
+            ModelGroup([
+                Model('stepfun-ai/GOT-OCR-2.0-hf', 'stepfun-ai/GOT-OCR-2.0-hf'),
+            ]),
+        ],
+        TemplateType.got_ocr2_hf,
+        get_model_tokenizer_got_ocr2_hf,
+        model_arch=ModelArch.got_ocr2_hf,
+        architectures=['GOTQwenForCausalLM'],
+        tags=['vision']))
+
+
+def get_model_tokenizer_step_audio(*args, **kwargs):
+    local_repo_path = kwargs.get('local_repo_path')
+    if not local_repo_path:
+        local_repo_path = git_clone_github('https://github.com/stepfun-ai/Step-Audio.git')
+    sys.path.append(local_repo_path)
+    from tokenizer import StepAudioTokenizer
+    encoder_path = safe_snapshot_download('stepfun-ai/Step-Audio-Tokenizer', check_local=True)
+    model, tokenizer = get_model_tokenizer_with_flash_attn(*args, **kwargs)
+    if model is not None:
+        model.encoder = StepAudioTokenizer(encoder_path)
+        # from tts import StepAudioTTS
+        # if not os.path.exists('speakers'):
+        #     shutil.copytree(os.path.join(local_repo_path, 'speakers'), 'speakers')
+        # decoder_path = safe_snapshot_download('stepfun-ai/Step-Audio-TTS-3B', check_local=True)
+        # model.decoder = StepAudioTTS(decoder_path, model.encoder)
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.step_audio, [
+            ModelGroup([
+                Model('stepfun-ai/Step-Audio-Chat', 'stepfun-ai/Step-Audio-Chat'),
+            ]),
+        ],
+        TemplateType.step_audio,
+        get_model_tokenizer_step_audio,
+        architectures=['Step1ForCausalLM'],
+        requires=['funasr', 'sox', 'conformer', 'openai-whisper', 'librosa'],
+        tags=['audio']))

swift/llm/model/model/telechat.py

@@ -0,0 +1,59 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from transformers import GenerationConfig
+
+from swift.llm import TemplateType
+from ..constant import LLMModelType
+from ..model_arch import ModelArch
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+
+
+def get_model_tokenizer_telechat(*args, **kwargs):
+    model, tokenizer = get_model_tokenizer_with_flash_attn(*args, **kwargs)
+    model_dir = args[0]
+    generation_config = GenerationConfig.from_pretrained(model_dir)
+    for k in ['bos_token_id', 'eos_token_id', 'pad_token_id', 'user_token_id', 'bot_token_id']:
+        setattr(tokenizer, k, getattr(generation_config, k))
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        LLMModelType.telechat,
+        [
+            ModelGroup([
+                Model('TeleAI/TeleChat-7B', 'Tele-AI/telechat-7B'),
+                Model('TeleAI/TeleChat-12B', 'Tele-AI/TeleChat-12B'),
+                Model('TeleAI/TeleChat-12B-v2', 'Tele-AI/TeleChat-12B-v2'),
+                Model('TeleAI/TeleChat-52B', 'TeleAI/TeleChat-52B'),
+            ]),
+            ModelGroup([
+                Model('swift/TeleChat-12B-V2-GPTQ-Int4'),
+            ]),
+            ModelGroup([
+                Model('TeleAI/TeleChat2-35B', 'Tele-AI/TeleChat2-35B'),
+                Model('TeleAI/TeleChat2-115B', 'Tele-AI/TeleChat2-115B'),
+            ]),
+        ],
+        TemplateType.telechat,
+        get_model_tokenizer_telechat,
+        model_arch=ModelArch.telechat,
+        architectures=['TelechatForCausalLM', 'TeleChatForCausalLM'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.telechat2,
+        [
+            ModelGroup([
+                Model('TeleAI/TeleChat2-3B', 'Tele-AI/TeleChat2-3B'),
+                Model('TeleAI/TeleChat2-7B-32K', 'Tele-AI/TeleChat2-7B-32K'),
+                Model('TeleAI/TeleChat2-35B-32K', 'Tele-AI/TeleChat2-35B-32K'),
+                Model('TeleAI/TeleChat2-35B-Nov', 'Tele-AI/TeleChat2-35B-Nov'),
+            ]),
+        ],
+        TemplateType.telechat2,
+        get_model_tokenizer_telechat,
+        model_arch=ModelArch.telechat,
+        architectures=['TeleChat2ForCausalLM'],
+    ))

swift/llm/model/model/valley.py

@@ -0,0 +1,82 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import sys
+from functools import partial, wraps
+from typing import Any, Dict
+
+from swift.llm import TemplateType
+from ..constant import MLLMModelType
+from ..model_arch import ModelArch
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import ModelInfo, git_clone_github, safe_snapshot_download
+
+
+def get_model_tokenizer_valley(model_dir: str,
+                               model_info: ModelInfo,
+                               model_kwargs: Dict[str, Any],
+                               load_model: bool = True,
+                               **kwargs):
+    llm_model_type = kwargs.pop('llm_model_type')
+    local_repo_path = kwargs.get('local_repo_path')
+    if not local_repo_path:
+        repo_path = 'https://github.com/bytedance/Valley.git'
+        local_repo_path = git_clone_github(repo_path)
+    sys.path.append(local_repo_path)
+
+    if llm_model_type == 'valley':
+        from transformers.modeling_outputs import CausalLMOutputWithPast
+        from valley_eagle.model.language_model.valley_qwen2 import ValleyQwen2ForCausalLM, ValleyConfig
+        model_config = ValleyConfig.from_pretrained(model_dir)
+        model_config.mm_vision_tower = safe_snapshot_download(
+            'AI-ModelScope/siglip-so400m-patch14-384', check_local=True)
+        model_config.eagle_vision_tower = safe_snapshot_download('Qwen/Qwen2-VL-7B-Instruct', check_local=True)
+        automodel_class = ValleyQwen2ForCausalLM
+
+        if not hasattr(ValleyQwen2ForCausalLM, '_origin_forward'):
+            forward = ValleyQwen2ForCausalLM.forward
+            ValleyQwen2ForCausalLM._origin_forward = forward
+
+            @wraps(forward)
+            def new_forward(*args, **kwargs):
+                import torch
+                outputs = forward(*args, **kwargs)
+                loss = outputs.loss
+                if loss is not None and loss.shape[-1] > 0:
+                    loss = torch.mean(loss, dim=-1)
+                return CausalLMOutputWithPast(
+                    loss=loss,
+                    logits=outputs.logits,
+                    past_key_values=outputs.past_key_values,
+                    hidden_states=outputs.hidden_states,
+                    attentions=outputs.attentions,
+                )
+
+            ValleyQwen2ForCausalLM.forward = new_forward
+    kwargs['model_config'] = model_config
+    kwargs['automodel_class'] = automodel_class
+    model, tokenizer = get_model_tokenizer_with_flash_attn(model_dir, model_info, model_kwargs, load_model, **kwargs)
+    if model is not None:
+        model.generation_config.repetition_penalty = 1.0  # Otherwise, Error. Same for original code.
+        from transformers import AutoProcessor, SiglipImageProcessor
+        tokenizer.image_processor = SiglipImageProcessor.from_pretrained(model.config.mm_vision_tower)
+        tokenizer.qwen2vl_processor = AutoProcessor.from_pretrained(
+            model.config.eagle_vision_tower, max_pixels=1280 * 28 * 28)
+        tokenizer.image_processor.crop_size = tokenizer.image_processor.size['height']
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.valley,
+        [
+            ModelGroup([
+                Model('bytedance-research/Valley-Eagle-7B'),
+            ], ),
+        ],
+        TemplateType.valley,
+        partial(get_model_tokenizer_valley, llm_model_type='valley'),
+        architectures=['ValleyQwen2ForCausalLM'],
+        model_arch=ModelArch.valley,
+        requires=['transformers>=4.42', 'av'],
+        tags=['vision'],
+    ))

swift/llm/model/model/yi.py

@@ -0,0 +1,146 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import sys
+from typing import Any, Dict
+
+from transformers import AutoTokenizer
+
+from swift.llm import TemplateType
+from swift.utils import get_logger
+from ..constant import LLMModelType, MLLMModelType
+from ..model_arch import ModelArch
+from ..register import Model, ModelGroup, ModelMeta, get_model_tokenizer_with_flash_attn, register_model
+from ..utils import ModelInfo, git_clone_github
+
+logger = get_logger()
+
+
+def get_model_tokenizer_yi(model_dir, *args, **kwargs):
+    tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True, use_fast=False)
+    return get_model_tokenizer_with_flash_attn(model_dir, *args, tokenizer=tokenizer, **kwargs)
+
+
+def get_model_tokenizer_yi_vl(model_dir: str,
+                              model_info: ModelInfo,
+                              model_kwargs: Dict[str, Any],
+                              load_model: bool = True,
+                              **kwargs):
+    local_repo_path = kwargs.get('local_repo_path')
+    if not local_repo_path:
+        local_repo_path = git_clone_github('https://github.com/01-ai/Yi')
+    sys.path.append(os.path.join(local_repo_path, 'VL'))
+    from llava.model import LlavaLlamaForCausalLM, LlavaConfig
+    from llava.model.constants import key_info
+
+    model_config = LlavaConfig.from_pretrained(model_dir)
+    mm_vision_tower = model_config.mm_vision_tower
+    model_config.mm_vision_tower = os.path.join(model_dir, *mm_vision_tower.rsplit('/', maxsplit=2)[-2:])
+    model_config.attention_dropout = 0.
+    key_info['model_path'] = model_dir
+    kwargs['automodel_class'] = kwargs['automodel_class'] or LlavaLlamaForCausalLM
+    model, tokenizer = get_model_tokenizer_yi(
+        model_dir, model_info, model_kwargs, load_model, model_config=model_config, **kwargs)
+    if model is not None:
+        logger.info('Please ignore the above warning.')
+        logger.info('Loading the parameters of vision_tower...')
+        model.resize_token_embeddings(len(tokenizer))
+        vision_tower = model.get_vision_tower()
+        vision_tower.load_model()
+        vision_tower.to(device=model.device, dtype=model_config.torch_dtype)
+        if not hasattr(model.config, 'max_sequence_length'):
+            model.config.max_sequence_length = 2048
+        tokenizer.image_processor = vision_tower.image_processor
+    return model, tokenizer
+
+
+register_model(
+    ModelMeta(
+        MLLMModelType.yi_vl,
+        [
+            ModelGroup([
+                Model('01ai/Yi-VL-6B', '01-ai/Yi-VL-6B'),
+                Model('01ai/Yi-VL-34B', '01-ai/Yi-VL-34B'),
+            ], ),
+        ],
+        TemplateType.yi_vl,
+        get_model_tokenizer_yi_vl,
+        model_arch=ModelArch.llava_llama,
+        architectures=['LlavaLlamaForCausalLM'],
+        requires=['transformers>=4.34'],
+        tags=['vision'],
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.yi,
+        [  # yi
+            ModelGroup([
+                Model('01ai/Yi-6B', '01-ai/Yi-6B'),
+                Model('01ai/Yi-6B-200K', '01-ai/Yi-6B-200K'),
+                Model('01ai/Yi-6B-Chat', '01-ai/Yi-6B-Chat'),
+                Model('01ai/Yi-6B-Chat-4bits', '01-ai/Yi-6B-Chat-4bits'),
+                Model('01ai/Yi-6B-Chat-8bits', '01-ai/Yi-6B-Chat-8bits'),
+                Model('01ai/Yi-9B', '01-ai/Yi-9B'),
+                Model('01ai/Yi-9B-200K', '01-ai/Yi-9B-200K'),
+                Model('01ai/Yi-34B', '01-ai/Yi-34B'),
+                Model('01ai/Yi-34B-200K', '01-ai/Yi-34B-200K'),
+                Model('01ai/Yi-34B-Chat', '01-ai/Yi-34B-Chat'),
+                Model('01ai/Yi-34B-Chat-4bits', '01-ai/Yi-34B-Chat-4bits'),
+                Model('01ai/Yi-34B-Chat-8bits', '01-ai/Yi-34B-Chat-8bits'),
+            ]),
+            # yi1.5
+            ModelGroup([
+                Model('01ai/Yi-1.5-6B', '01-ai/Yi-1.5-6B'),
+                Model('01ai/Yi-1.5-6B-Chat', '01-ai/Yi-1.5-6B-Chat'),
+                Model('01ai/Yi-1.5-9B', '01-ai/Yi-1.5-9B'),
+                Model('01ai/Yi-1.5-9B-Chat', '01-ai/Yi-1.5-9B-Chat'),
+                Model('01ai/Yi-1.5-9B-Chat-16K', '01-ai/Yi-1.5-9B-Chat-16K'),
+                Model('01ai/Yi-1.5-34B', '01-ai/Yi-1.5-34B'),
+                Model('01ai/Yi-1.5-34B-Chat', '01-ai/Yi-1.5-34B-Chat'),
+                Model('01ai/Yi-1.5-34B-Chat-16K', '01-ai/Yi-1.5-34B-Chat-16K'),
+            ]),
+            # yi1.5-quant
+            ModelGroup([
+                Model('AI-ModelScope/Yi-1.5-6B-Chat-GPTQ', 'modelscope/Yi-1.5-6B-Chat-GPTQ'),
+                Model('AI-ModelScope/Yi-1.5-6B-Chat-AWQ', 'modelscope/Yi-1.5-6B-Chat-AWQ'),
+                Model('AI-ModelScope/Yi-1.5-9B-Chat-GPTQ', 'modelscope/Yi-1.5-9B-Chat-GPTQ'),
+                Model('AI-ModelScope/Yi-1.5-9B-Chat-AWQ', 'modelscope/Yi-1.5-9B-Chat-AWQ'),
+                Model('AI-ModelScope/Yi-1.5-34B-Chat-GPTQ', 'modelscope/Yi-1.5-34B-Chat-GPTQ'),
+                Model('AI-ModelScope/Yi-1.5-34B-Chat-AWQ', 'modelscope/Yi-1.5-34B-Chat-AWQ'),
+            ]),
+        ],
+        TemplateType.chatml,
+        get_model_tokenizer_yi,
+        architectures=['LlamaForCausalLM'],
+        model_arch=ModelArch.llama,
+    ))
+
+register_model(
+    ModelMeta(
+        LLMModelType.yi_coder, [
+            ModelGroup([
+                Model('01ai/Yi-Coder-1.5B', '01-ai/Yi-Coder-1.5B'),
+                Model('01ai/Yi-Coder-9B', '01-ai/Yi-Coder-9B'),
+                Model('01ai/Yi-Coder-1.5B-Chat', '01-ai/Yi-Coder-1.5B-Chat'),
+                Model('01ai/Yi-Coder-9B-Chat', '01-ai/Yi-Coder-9B-Chat'),
+            ], )
+        ],
+        TemplateType.yi_coder,
+        get_model_tokenizer_yi,
+        architectures=['LlamaForCausalLM'],
+        model_arch=ModelArch.llama,
+        tags=['coding']))
+
+register_model(
+    ModelMeta(
+        LLMModelType.sus,
+        [
+            ModelGroup([
+                Model('SUSTC/SUS-Chat-34B', 'SUSTech/SUS-Chat-34B'),
+            ]),
+        ],
+        TemplateType.sus,
+        get_model_tokenizer_yi,
+        model_arch=ModelArch.llama,
+        architectures=['LlamaForCausalLM'],
+    ))

swift/llm/sampling/__init__.py

@@ -0,0 +1 @@
+from .sampling import sampling_main

swift/llm/sampling/base.py

@@ -0,0 +1,55 @@
+from typing import Any, Dict, List
+
+from swift.llm import SamplingArguments
+from swift.plugin import orms, prms
+from swift.utils import get_logger
+
+logger = get_logger()
+
+
+class Sampler:
+
+    def __init__(self, input_args: SamplingArguments):
+        self.args = input_args
+        self.template = None
+        self.processor = None
+        self.prm_model = None
+        self.orm_model = None
+        self._prepare_model_tokenizer()
+        self._prepare_template()
+        self._prepare_rm()
+
+    def _prepare_model_tokenizer(self):
+        args = self.args
+        _, self.processor = args.get_model_processor(load_model=False)
+
+    def _prepare_rm(self):
+        if self.args.prm_model is None:
+            self.prm_model = None
+            logger.warning('prm_model is None.')
+        elif self.args.prm_model in prms:
+            self.prm_model = prms[self.args.prm_model]()
+        else:
+            from swift.llm import PtEngine
+            self.prm_model = PtEngine(self.args.prm_model, max_batch_size=64)
+
+        if self.args.orm_model is None:
+            self.orm_model = None
+            logger.warning('orm_model is None.')
+        elif self.args.orm_model in orms:
+            self.orm_model = orms[self.args.orm_model]()
+        else:
+            from swift.llm import PtEngine
+            self.orm_model = PtEngine(self.args.orm_model, max_batch_size=64)
+
+    def _prepare_template(self) -> None:
+        template = self.args.get_template(self.processor)
+        self.template = template
+        self.template.set_mode('train')
+
+    def truncate_input(self, slices: List[Dict[str, Any]]):
+        """Truncate the input rows to avoid hitting the max length of the policy model"""
+        return slices
+
+    def do_sample(self, data):
+        raise NotImplementedError

swift/llm/sampling/distill_sampler.py

@@ -0,0 +1,148 @@
+import os
+from copy import deepcopy
+from typing import List, Optional
+
+from openai import OpenAI
+
+from swift.llm.infer.protocol import InferRequest, RequestConfig
+from swift.llm.sampling.vanilla_sampler import VanillaSampler
+from .utils import get_messages_md5
+
+
+class OpenAI_Engine():
+
+    def __init__(
+        self,
+        model: str,
+        stream: bool = False,
+        base_url: str = 'https://dashscope.aliyuncs.com/compatible-mode/v1',
+        api_key: str = '',
+        **kwargs,
+    ):
+        self.model = model
+        self.stream = stream
+        self.client = OpenAI(api_key=api_key if api_key else os.getenv('OPENAI_API_KEY'), base_url=base_url, **kwargs)
+
+    def infer(
+        self,
+        infer_requests: List[InferRequest],
+        request_config: Optional[RequestConfig] = None,
+    ):
+        resp_contents = []
+        for infer_request in infer_requests:
+            completion = self.client.chat.completions.create(
+                model=self.model,
+                messages=infer_request['messages'],
+                temperature=request_config.temperature,
+                top_p=request_config.top_p,
+                max_tokens=request_config.max_tokens,
+                stream=self.stream,
+            )
+            if self.stream:
+                reasoning_content = ''
+                content = ''
+                for chunk in completion:
+                    chunk_choices = chunk.choices
+                    if len(chunk_choices) == 0:
+                        continue
+                    reasoning_chunk = chunk_choices[0].delta.reasoning_content if hasattr(
+                        chunk_choices[0].delta, 'reasoning_content') else ''
+                    answer_chunk = chunk_choices[0].delta.content
+                    if reasoning_chunk:
+                        reasoning_content += reasoning_chunk
+                    elif answer_chunk:
+                        content += answer_chunk
+            else:
+                if hasattr(completion.choices[0].message, 'reasoning_content'):
+                    reasoning_content = completion.choices[0].message.reasoning_content
+                content = completion.choices[0].message.content
+            assert len(content) > 0, 'Empty completion'
+            if reasoning_content:
+                resp_content = f'<think>{reasoning_content}</think>\n\n<answer>{content}</answer>'
+            else:
+                resp_content = content
+            resp_contents.append(resp_content)
+
+        return resp_contents
+
+
+class DistillSampler(VanillaSampler):
+
+    def __init__(self, *args, **kwargs):
+        super(VanillaSampler, self).__init__(*args, **kwargs)
+        assert self.args.sampler_engine == 'client'
+        _Engine = OpenAI_Engine
+        self.infer_engine = _Engine(model=self.args.model, stream=self.args.stream, **self.args.engine_kwargs)
+        self.infer_engine.strict = False
+        self.caches = self.read_cache()
+
+    def _prepare_model_tokenizer(self):
+        pass
+
+    def _prepare_template(self):
+        pass
+
+    def extract_choice(self, resp):
+        message = resp.choices[0].message
+        if hasattr(message, 'reasoning_content'):
+            reps_content = f'<think>{message.reasoning_content}</think>\n\n<answer>{message.content}</answer>'
+        else:
+            reps_content = message.content
+        return reps_content
+
+    def generate(self, data):
+        resp_all = []
+        infer_requests = []
+        sent = 0
+        rows = self.convert_data_to_rows(data)
+        for idx, row in enumerate(rows):
+            row = deepcopy(row)
+            messages = row['messages']
+            uuid = get_messages_md5(row)
+            if uuid in self.caches:
+                choices = self.caches[uuid]['choices']
+                if len(choices) == self.args.num_return_sequences:
+                    continue
+            if self.args.system:
+                if messages[0]['role'] == 'system':
+                    messages[0]['content'] = self.args.system
+                else:
+                    messages.insert(0, {'role': 'system', 'content': self.args.system})
+            if messages[-1]['role'] == 'assistant':
+                messages = messages[:-1]
+
+            row['messages'] = messages
+            infer_request = row
+            for i in range(self.args.num_return_sequences):
+                infer_requests.append(deepcopy(infer_request))
+            sent += 1
+
+        request_config = RequestConfig(
+            max_tokens=self.args.max_new_tokens,
+            temperature=self.args.temperature,
+            top_k=self.args.top_k,
+            top_p=self.args.top_p,
+        )
+
+        resp_list = []
+        if len(infer_requests) > 0:
+            resp_list = self.infer_engine.infer(infer_requests, request_config=request_config)
+
+        _cur = 0
+        for idx, row in enumerate(rows):
+            row = deepcopy(row)
+            uuid = get_messages_md5(row)
+            if uuid in self.caches:
+                choices = self.caches[uuid]['choices']
+                if len(choices) == self.args.num_return_sequences:
+                    row['choices'] = choices
+                    resp_all.append(row)
+                    continue
+
+            resps = row
+            resps['choices'] = []
+            for j in range(self.args.num_return_sequences * _cur, self.args.num_return_sequences * (_cur + 1)):
+                resps['choices'].append(resp_list[j])
+            resp_all.append(resps)
+            _cur += 1
+        return resp_all

swift/llm/sampling/mcts.py

@@ -0,0 +1,400 @@
+import time
+import traceback
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from copy import deepcopy
+
+import json
+import numpy as np
+
+from swift.llm import InferRequest, SamplingArguments
+from swift.llm.infer.protocol import UsageInfo
+from swift.utils import get_logger
+from .base import Sampler
+from .utils import get_reward, perform_infer
+
+logger = get_logger()
+
+NXT_PROMPT = """Continue.
+"""
+
+next_message = {
+    'role': 'user',
+    'content': NXT_PROMPT,
+}
+
+
+class LanguageNode:
+
+    def __init__(
+        self,
+        step: str = None,
+        sep_token: str = None,
+        parent: 'LanguageNode' = None,
+    ):
+        self.parent = parent
+
+        if sep_token:
+            self.sep_token = sep_token
+        else:
+            self.sep_token = parent.sep_token
+
+        if parent:
+            self.path = parent.path[:] + [step]
+            self.answer = parent.answer + step + self.sep_token
+            self.depth = parent.depth + 1
+        else:
+            self.path = []
+            self.answer = ''
+            self.depth = 0
+
+        self.active_children = []
+        self.children = []
+        self.visit_count = 0
+        self.process_reward = 0.0
+        self.outcome_reward = 0.0
+        self.terminated = False
+        self.correct = False
+
+    def is_leaf(self):
+        return len(self.children) == 0
+
+    def is_root(self):
+        return self.parent is None
+
+    def visit(self):
+        self.visit_count += 1
+
+    def init_and_update_value(self, value):
+        self.outcome_reward = (self.outcome_reward * self.visit_count + value) / (self.visit_count + 1)
+
+    def add_child(self, child: 'LanguageNode'):
+        self.children.append(child)
+        if not child.terminated:
+            self.active_children.append(child)
+
+    def collect(self):
+        result = {
+            'path': self.path,
+            'depth': self.depth,
+            'visit_count': self.visit_count,
+            'process_reward': self.process_reward,
+            'outcome_reward': self.outcome_reward,
+            'terminated': str(self.terminated),
+            'correct': str(self.correct),
+            'children': [child.collect() for child in self.children],
+        }
+        return result
+
+    def __lt__(self, other):
+        return self.outcome_reward < other.outcome_reward
+
+
+class MctsSampler(Sampler):
+
+    def __init__(self, input_args: SamplingArguments):
+        super().__init__(input_args)
+        self.usage_info = UsageInfo(0, 0, 0)
+
+    def _prepare_model_tokenizer(self):
+        args = self.args
+        self.infer_kwargs = {}
+        if args.sampler_engine == 'client':
+            from swift.llm import InferClient
+            api_key = args.api_key
+            base_url = args.base_url
+            self.infer_engine = [
+                InferClient(base_url=base_url, api_key=api_key) for _ in range(args.num_return_sequences)
+            ]
+            self.infer_kwargs['model'] = args.model
+        else:
+            _Engine = self.get_infer_engine()
+            self.infer_engine = _Engine(self.args.model, model_type=self.args.model_type, **self.args.engine_kwargs)
+
+    def get_infer_engine(self):
+        if self.args.sampler_engine == 'pt':
+            from swift.llm import PtEngine
+            _Engine = PtEngine
+        elif self.args.sampler_engine == 'vllm':
+            from swift.llm import VllmEngine
+            _Engine = VllmEngine
+        elif self.args.sampler_engine == 'lmdeploy':
+            from swift.llm import LmdeployEngine
+            _Engine = LmdeployEngine
+        elif self.args.sampler_engine == 'no':
+            _Engine = None
+        else:
+            raise ValueError(f'Cannot find engine name: {self.args.sampler_engine}')
+        return _Engine
+
+    def _prepare_template(self) -> None:
+        # Hack from super()
+        self._prepare_request_configs()
+
+    def _prepare_request_configs(self):
+        _args = self.args
+        request_config = _args.get_request_config()
+        request_config.stop = _args.stop_words
+        request_config.seed = _args.seed
+        self.expand_request_configs = []
+        self.rollout_request_configs = []
+        for i in range(_args.num_return_sequences):
+            expand_request_config = deepcopy(request_config)
+            expand_request_config.n = 1
+            expand_request_config.num_beams = expand_request_config.n
+            expand_request_config.seed += i
+            self.expand_request_configs.append(expand_request_config)
+            rollout_request_config = deepcopy(request_config)
+            rollout_request_config.max_tokens = 500
+            rollout_request_config.temperature = 0.0
+            rollout_request_config.n = 1
+            self.rollout_request_configs.append(rollout_request_config)
+
+    def update_usage_info(self, response):
+        for key, value in self.usage_info.__dict__.items():
+            update_value = getattr(response.usage, key, None) + value
+            setattr(self.usage_info, key, update_value)
+
+    def search_single(self, query, ground_truth):
+
+        def _uct(uct_curr_node: LanguageNode):
+            alpha = _args.process_reward_rate
+            value = alpha * uct_curr_node.process_reward + (1 - alpha) * uct_curr_node.outcome_reward
+            if uct_curr_node.is_root():
+                return value
+
+            exploitation_score = value
+            exploration_score = (
+                _args.exploration_rate
+                * np.sqrt(np.log(uct_curr_node.parent.visit_count + 1) / (uct_curr_node.visit_count + 1)))
+
+            return exploration_score + exploitation_score
+
+        def _select(select_curr_node: LanguageNode):
+            while not select_curr_node.is_leaf():
+                select_curr_node = max(select_curr_node.active_children, key=lambda x: _uct(x))
+            return select_curr_node
+
+        def _expand(expand_curr_node: LanguageNode):
+            n = _args.num_return_sequences - len(expand_curr_node.children)
+            if expand_curr_node.is_root():
+                infer_requests = [InferRequest(system_message + [prompt_message]) for _ in range(n)]
+            else:
+                history_message = {
+                    'role': 'assistant',
+                    'content': expand_curr_node.answer,
+                }
+                infer_request = InferRequest(system_message + [prompt_message, history_message, next_message])
+                infer_requests = [infer_request for _ in range(n)]
+
+            # e_time = time.time()
+            # To perform the Expand operation in parallel,
+            # there's no need to consider the order for now, since the Prompt is the same.
+            expand_iter_index = 0
+            while True:
+                responses = perform_infer(self.infer_engine, infer_requests, self.expand_request_configs,
+                                          **self.infer_kwargs)
+                if len(responses) > 0:
+                    break
+                if expand_iter_index == 5:
+                    raise ValueError('Expand should not return any response')
+                expand_iter_index += 1
+            # logger.info(f"expand.expand time: {time.time() - e_time}")
+
+            # To fetch Outcome Reward in parallel,
+            # the Outcome-Reward obtained is returned in order, so they can be directly matched accordingly.
+            orm_infer_requests = []
+            unique_output = set()
+            for response in responses:
+                self.update_usage_info(response)
+                output = response.choices[0].message.content.rstrip(sep_token + '\n').split(sep_token)[0]
+                if output in unique_output:
+                    continue
+                unique_output.add(output)
+                orm_infer_requests.append(InferRequest([{'role': 'assistant', 'content': output}]))
+                child = LanguageNode(step=output, parent=expand_curr_node)
+                if self.orm_model.check_terminate(child.answer)[0]:
+                    child.terminated = True
+                expand_curr_node.add_child(child)
+
+            # e_time = time.time()
+            orm_score, _orm_mask = get_reward(
+                self.orm_model,
+                orm_infer_requests,
+                ground_truths=[ground_truth] * len(orm_infer_requests),
+                threshold=0.0)
+            # logger.info(f"expand.orm time: {time.time() - e_time}")
+            for child, score in zip(expand_curr_node.children, orm_score):
+                if child.terminated:
+                    child.init_and_update_value(score)
+                    child.correct = score > 0.9
+                    terminated_nodes.append(child)
+
+            # e_time = time.time()
+            if self.prm_model:
+                prm_infer_requests = []
+                for child in expand_curr_node.children:
+                    prm_message = {'role': 'assistant', 'content': child.answer}
+                    prm_infer_requests.append(InferRequest([prompt_message, prm_message]))
+                prm_score, _prm_mask = get_reward(
+                    self.prm_model,
+                    prm_infer_requests,
+                    ground_truths=[ground_truth] * len(prm_infer_requests),
+                    threshold=0.0)
+                for child, score in zip(expand_curr_node.children, prm_score):
+                    child.process_reward = score
+            # logger.info(f"expand.prm time: {time.time() - e_time}")
+
+        def _rollout(rollout_curr_node: LanguageNode):
+            rollout_depth = 0
+            rollout_nodes = {}
+            for i in range(len(rollout_curr_node.active_children)):
+                rollout_nodes[i] = {
+                    'node': rollout_curr_node.active_children[i],
+                    'history_messages': {
+                        'role': 'assistant',
+                        'content': rollout_curr_node.active_children[i].answer,
+                    },
+                }
+            active_rollout_nodes = list(rollout_nodes.keys())
+            while len(active_rollout_nodes) > 0 and rollout_depth < _args.rollout_depth:
+                # r_time = time.time()
+                infer_requests = [
+                    InferRequest(system_message
+                                 + [prompt_message, rollout_nodes[index]['history_messages'], next_message])
+                    for index in active_rollout_nodes
+                ]
+                # logger.info(f"rollout.prepare time: {time.time() - r_time}")
+                # r_time = time.time()
+                rollout_iter_index = 0
+                while True:
+                    responses = perform_infer(self.infer_engine, infer_requests, self.rollout_request_configs,
+                                              **self.infer_kwargs)
+                    if len(responses) > 0:
+                        break
+                    if rollout_iter_index == 5:
+                        raise ValueError('Rollout should not return any response')
+                    rollout_iter_index += 1
+                # logger.info(f"rollout.infer time: {time.time() - r_time}")
+
+                # r_time = time.time()
+                orm_infer_requests = []
+                end_paths = []
+                for index, response in zip(active_rollout_nodes, responses):
+                    self.update_usage_info(response)
+                    output = response.choices[0].message.content.rstrip(sep_token
+                                                                        + '\n').split(sep_token)[0] + sep_token + '\n'
+                    rollout_nodes[index]['history_messages']['content'] += output
+                    end_paths.append(rollout_nodes[index]['history_messages']['content'])
+                    orm_infer_requests.append(InferRequest([rollout_nodes[index]['history_messages']]))
+                # logger.info(f"rollout.orm_prepare time: {time.time() - r_time}")
+
+                # r_time = time.time()
+                orm_score, _orm_mask = get_reward(
+                    self.orm_model,
+                    orm_infer_requests,
+                    ground_truths=[ground_truth] * len(infer_requests),
+                    threshold=0.0)
+                # logger.info(f"rollout.get_orm time: {time.time() - r_time}")
+                terminated_state = self.orm_model.check_terminate(end_paths)
+                for index, score, terminated in zip(active_rollout_nodes, orm_score, terminated_state):
+                    if terminated:
+                        rollout_curr_node.active_children[index].init_and_update_value(score)
+                        if score > 0.9:
+                            rollout_correct_answers.append(rollout_nodes[index]['history_messages']['content'])
+                        else:
+                            rollout_incorrect_answers.append(rollout_nodes[index]['history_messages']['content'])
+                        rollout_nodes.pop(index)
+                active_rollout_nodes = list(rollout_nodes.keys())
+                rollout_depth += 1
+
+        def _back_propagate(back_curr_node: LanguageNode):
+            while back_curr_node:
+                if back_curr_node == curr_node:
+                    best_child_value = max([child.outcome_reward for child in back_curr_node.children])
+                    back_curr_node.init_and_update_value(best_child_value)
+                    last_child_value = back_curr_node.outcome_reward
+                else:
+                    back_curr_node.init_and_update_value(last_child_value)
+                    last_child_value = back_curr_node.outcome_reward
+                back_curr_node.visit()
+                if len(back_curr_node.active_children) == 0:
+                    back_curr_node.terminated = True
+                    if not back_curr_node.is_root():
+                        back_curr_node.parent.active_children.remove(back_curr_node)
+                back_curr_node = back_curr_node.parent
+
+        _args = self.args
+        system_message = [] + _args.system_message
+        sep_token = _args.stop_words[0] + '\n'
+        _root = LanguageNode(sep_token=sep_token)
+        prompt_message = {
+            'role': 'user',
+            'content': query,
+        }
+
+        rollout_correct_answers, rollout_incorrect_answers, terminated_nodes = [], [], []
+        iter_count = 0
+        stop_reason = None
+        while True:
+            logger.info(f'iter_count: {iter_count}' + '.' * 10)
+            s_time = time.time()
+            curr_node = _select(_root)
+            logger.debug('select' + '=' * 10 + f'time: {time.time() - s_time}')
+            s_time = time.time()
+            _expand(curr_node)
+            logger.debug('expand' + '=' * 10 + f'time: {time.time() - s_time}')
+            if curr_node.depth > _args.rollout_start_depth:
+                s_time = time.time()
+                _rollout(curr_node)
+                logger.debug('rollout' + '=' * 10 + f'time: {time.time() - s_time}')
+            s_time = time.time()
+            _back_propagate(curr_node)
+            logger.debug('back propagate' + '=' * 10 + f'time: {time.time() - s_time}')
+            if len(rollout_correct_answers) + len(rollout_incorrect_answers) >= 2 * _args.num_return_sequences:
+                if 4 * len(rollout_incorrect_answers) < len(rollout_correct_answers):
+                    stop_reason = 'too easy'
+                    break
+                elif 4 * len(rollout_correct_answers) < len(rollout_incorrect_answers):
+                    stop_reason = 'too hard'
+                    break
+            if _root.terminated:
+                stop_reason = 'root terminated'
+                break
+            if len(terminated_nodes) >= _args.num_return_sequences:
+                stop_reason = 'enough nodes'
+                break
+            if iter_count >= _args.max_iterations:
+                stop_reason = 'max_iterations'
+                break
+            iter_count += 1
+        logger.info(f'stop_reason: {stop_reason}')
+        # logger.info(f"rollout_correct_answers: {rollout_correct_answers}")
+        # logger.info(f"rollout_incorrect_answers: {rollout_incorrect_answers}")
+
+        monte_carlo_tree = _root.collect()
+        result = {
+            'query': query,
+            'ground_truth': ground_truth,
+            'rollout_correct_answers': rollout_correct_answers,
+            'rollout_incorrect_answers': rollout_incorrect_answers,
+            'monte_carlo_tree': monte_carlo_tree,
+        }
+        result_json = json.dumps(result, ensure_ascii=False)
+        logger.info(result_json)
+        return result_json
+
+    def do_sample(self, data):
+        if not isinstance(data, list):
+            data = [data]
+        generated = []
+        for item in data:
+            logger.info(f'time: {time.ctime(time.time())}')
+            try:
+                messages = item['messages'][0]
+                query = messages[0]['content']
+                ground_truth = messages[1]['content']
+                generated.append(self.search_single(query, ground_truth) + '\n')
+            except Exception as e:
+                logger.error(f'Error: {e}')
+                logger.error(f'Traceback: {traceback.format_exc()}')
+        return generated

swift/llm/sampling/sampling.py

@@ -0,0 +1,106 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import shutil
+import time
+from typing import List, Union
+
+import json
+
+from swift.llm import SamplingArguments, SwiftPipeline, load_dataset
+from swift.utils import get_logger
+
+logger = get_logger()
+
+
+class SwiftSampling(SwiftPipeline):
+    args_class = SamplingArguments
+    args: args_class
+
+    def __init__(self, args: Union[List[str], SamplingArguments, None] = None) -> None:
+        super().__init__(args)
+        self.args.save_args()
+        os.makedirs(self.args.output_dir, exist_ok=True)
+        self.cur_piece = 0
+        self.total_piece = 1
+
+        if self.args.data_range:
+            self.cur_piece, self.total_piece = self.args.data_range
+
+        if self.args.sampler_type == 'sample':
+            from swift.llm.sampling.vanilla_sampler import VanillaSampler
+            self.sampler = VanillaSampler(self.args)
+        elif self.args.sampler_type == 'mcts':
+            from swift.llm.sampling.mcts import MctsSampler
+            self.sampler = MctsSampler(self.args)
+        elif self.args.sampler_type == 'distill':
+            from swift.llm.sampling.distill_sampler import DistillSampler
+            self.sampler = DistillSampler(self.args)
+        else:
+            raise ValueError(f'Unsupported sampler type: {self.args.sampler_type}')
+
+    def _get_dataset(self):
+        args = self.args
+        dataset_kwargs = args.get_dataset_kwargs()
+        sampling_dataset, _ = load_dataset(
+            args.dataset, split_dataset_ratio=0., shuffle=args.dataset_shuffle, **dataset_kwargs)
+        logger.info(f'Sampling_dataset: {sampling_dataset}')
+        dataset_len = len(sampling_dataset)
+        piece_len = dataset_len // self.total_piece
+        sampling_dataset = sampling_dataset.select(range(piece_len * self.cur_piece, piece_len * (self.cur_piece + 1)))
+        return sampling_dataset
+
+    def run(self):
+        os.makedirs(self.args.output_dir, exist_ok=True)
+        iter_file = os.path.join(self.args.output_dir, self.args.output_file)
+        resume_file = os.path.join(self.args.output_dir, self.args.output_file + '.resume')
+        tmp_file = os.path.join(self.args.output_dir, self.args.output_file + '.tmp')
+        ckpt_state_file = os.path.join(self.args.output_dir, 'ckpt_state.json')
+        if os.path.exists(iter_file) and not self.args.override_exist_file:
+            return
+
+        index_resume = -1
+        write_mode = 'w'
+        if self.args.resume:
+            write_mode = 'a'
+            if os.path.exists(resume_file):
+                shutil.copyfile(resume_file, tmp_file)
+
+            if os.path.exists(ckpt_state_file):
+                with open(ckpt_state_file, 'r') as ckpt_state:
+                    data = json.load(ckpt_state)
+                    index_resume = data.get('index', -1)
+                    logger.info(f'Loaded index_resume: {index_resume}')
+        else:
+            if os.path.exists(tmp_file):
+                os.remove(tmp_file)
+
+        dataset = self._get_dataset()
+        dataset_len = len(dataset)
+        total_iters = int(dataset_len // self.args.num_sampling_per_gpu_batch_size)
+
+        if self.args.num_sampling_per_gpu_batches is None or self.args.num_sampling_per_gpu_batches > total_iters:
+            self.args.num_sampling_per_gpu_batches = total_iters
+
+        with open(tmp_file, write_mode) as f:
+            for _index in range(self.args.num_sampling_per_gpu_batches):
+                if _index <= index_resume:
+                    continue
+                logger.info(f' Sampling index:{_index}')
+                slices = dataset[self.args.num_sampling_per_gpu_batch_size
+                                 * _index:self.args.num_sampling_per_gpu_batch_size * (_index + 1)]
+                slices = self.sampler.truncate_input(slices)
+                generated = self.sampler.do_sample(slices)
+                f.writelines(generated)
+                f.flush()
+                shutil.copy(tmp_file, resume_file)
+                with open(ckpt_state_file, 'w') as ckpt_state:
+                    json.dump({'index': _index}, ckpt_state)
+
+        if os.path.exists(iter_file):
+            shutil.move(iter_file, iter_file + '.' + str(int(time.time())))
+        shutil.move(resume_file, iter_file)
+        logger.info(f'Sample file {iter_file} generated.')
+
+
+def sampling_main(args: Union[List[str], SamplingArguments, None] = None):
+    return SwiftSampling(args).main()

swift/llm/sampling/utils.py

@@ -0,0 +1,172 @@
+import hashlib
+import inspect
+from copy import copy
+from typing import Any, Dict, List, Optional
+
+import json
+import numpy as np
+
+from swift.llm import InferRequest, RequestConfig
+from swift.utils import get_logger
+
+logger = get_logger()
+
+
+def get_messages_md5(row: Dict[str, Any]):
+    row = copy(row)
+    row.pop('choices', None)
+    serialized = json.dumps(row, sort_keys=True)
+    return hashlib.md5(serialized.encode('utf-8')).hexdigest()
+
+
+def get_reward(model: Any,
+               infer_requests: List[InferRequest],
+               request_config: RequestConfig = None,
+               ground_truths: List[str] = None,
+               threshold: Optional[float] = None):
+    """Get reward from an RM model.
+
+    Args:
+        model: The model instance or an RM evaluator
+        infer_requests: Infer requests sent to the model
+        request_config: Infer config
+        ground_truths: The ground truth list
+        threshold: An optional threshold to generate the mask
+
+    Returns:
+        Tuple
+        Index 0: The min-max normalized scores matched the infer_requests
+        Index 1: The mask filtered by the threshold
+    """
+    from swift.llm import InferEngine
+    infer_func = model.infer if isinstance(model, InferEngine) else model.__call__
+    parameters = inspect.signature(infer_func).parameters
+    gt_param = {}
+    if 'ground_truths' in parameters:
+        gt_param = {'ground_truths': ground_truths}
+    if isinstance(infer_requests[0], dict):
+        infer_requests = [InferRequest(messages=req['messages']) for req in infer_requests]
+    rewards = infer_func(infer_requests, request_config=request_config, **gt_param)
+    from swift.llm.infer.protocol import ChatCompletionResponse
+    if isinstance(rewards[0], ChatCompletionResponse):
+        print('reward:', rewards[0].choices[0].message.content)
+        if isinstance(rewards[0].choices[0].message.content, str):
+            rewards = [float(r.choices[0].message.content.strip('[]')) for r in rewards]
+        elif isinstance(rewards[0].choices[0].message.content, list):
+            rewards = [float(min(r.choices[0].message.content)) for r in rewards]
+        else:
+            rewards = [float(r.choices[0].message.content) for r in rewards]
+    arr = []
+    for reward in rewards:
+        if isinstance(reward, (list, tuple)):
+            arr.append(min(reward))
+        else:
+            arr.append(float(reward))
+
+    _mask = np.array([True] * len(arr))
+    if threshold is not None:
+        # > not >=, orm caller passes 0, which will cause error
+        _mask = np.array([a > threshold for a in arr])
+
+    def normalize(arr):
+        min_val = np.min(arr)
+        max_val = np.max(arr)
+        if min_val == max_val:
+            if min_val == 0:
+                constant_value = 0.0
+            else:
+                constant_value = min(1.0, min_val)
+            return np.full_like(arr, fill_value=constant_value, dtype=np.float64)
+        normalized = (arr - min_val) / (max_val - min_val + 1e-5)
+        return normalized
+
+    return normalize(arr), _mask
+
+
+def perform_infer(infer_engines, infer_requests, request_configs, **infer_kwargs):
+    if isinstance(infer_engines, list):
+        assert len(infer_engines) >= len(request_configs) >= len(infer_requests)
+        from concurrent.futures import ThreadPoolExecutor, as_completed
+        n = len(infer_requests)
+        with ThreadPoolExecutor(max_workers=n) as executor:
+            futures = {
+                executor.submit(perform_infer, infer_engines[i], infer_requests[i], request_configs[i], **infer_kwargs):
+                i
+                for i in range(n)
+            }
+            responses = []
+            for future in as_completed(futures):
+                task_id = futures[future]
+                try:
+                    responses += future.result()
+                except Exception as e:
+                    logger.info(f'Perform infer task: {task_id} get an error: {e}')
+        return responses
+    elif isinstance(infer_requests, list):
+        responses = []
+        if isinstance(request_configs, list):
+            assert len(infer_requests) <= len(request_configs)
+            for i in range(len(infer_requests)):
+                responses += infer_engines.infer(
+                    [infer_requests[i]],
+                    request_configs[i],
+                    **infer_kwargs,
+                )
+        elif isinstance(request_configs, RequestConfig):
+            for infer_request in infer_requests:
+                responses += infer_engines.infer(
+                    [infer_request],
+                    request_configs,
+                    **infer_kwargs,
+                )
+        return responses
+    return infer_engines.infer(
+        [infer_requests],
+        request_configs,
+        **infer_kwargs,
+    )
+
+
+def collect_from_mct(monte_carlo_tree, collect_filter_threshold):
+    from transformers.utils import strtobool
+    if isinstance(monte_carlo_tree, str):
+        monte_carlo_tree = json.loads(monte_carlo_tree)
+
+    def _collect(collect_curr_node, _outcome_rewards: list[float], _process_rewards: list[float]):
+        _prefer_pairs, _correct_answers, _incorrect_answers = [], [], []
+        _outcome_rewards = _outcome_rewards[:] + [collect_curr_node['outcome_reward']]
+        _process_rewards = _process_rewards[:] + [collect_curr_node['process_reward']]
+        if len(collect_curr_node['children']) > 0:
+            for child in collect_curr_node['children']:
+                p, c, i = _collect(child, _outcome_rewards, _process_rewards)
+                _prefer_pairs += p
+                _correct_answers += c
+                _incorrect_answers += i
+            sorted_children = sorted(collect_curr_node['children'], key=lambda x: x['outcome_reward'])
+            if sorted_children[-1]['outcome_reward'] - sorted_children[0]['outcome_reward'] > collect_filter_threshold:
+                # TODO: filter with visit count
+                prefer_pair = {
+                    'path': 'ки\n'.join(collect_curr_node['path']),
+                    'good': sorted_children[-1]['path'][-1],
+                    'good_score': sorted_children[-1]['outcome_reward'],
+                    'bad': sorted_children[0]['path'][-1],
+                    'bad_score': sorted_children[0]['outcome_reward'],
+                }
+                _prefer_pairs.append(prefer_pair)
+        if strtobool(collect_curr_node['terminated']):
+            _answer = {
+                'answer': 'ки\n'.join(collect_curr_node['path']),
+                'mean_outcome_reward': np.mean(_outcome_rewards),
+                'min_outcome_reward': np.min(_outcome_rewards),
+                'mean_process_reward': np.mean(_process_rewards),
+                'min_process_reward': np.min(_process_rewards),
+            }
+            if strtobool(collect_curr_node['correct']):
+                _correct_answers.append(_answer)
+            else:
+                _incorrect_answers.append(_answer)
+        return _prefer_pairs, _correct_answers, _incorrect_answers
+
+    _root = monte_carlo_tree
+    prefer_pairs, correct_answers, incorrect_answers = _collect(_root, [], [])
+    return prefer_pairs, correct_answers, incorrect_answers

swift/llm/sampling/vanilla_sampler.py

@@ -0,0 +1,216 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from copy import deepcopy
+from typing import Any, Dict, List
+
+import json
+import numpy as np
+
+from swift.llm import RequestConfig
+from swift.llm.sampling.base import Sampler
+from swift.llm.template.template_inputs import InferRequest
+from swift.utils import get_logger
+from .utils import get_messages_md5, get_reward
+
+logger = get_logger()
+
+
+class VanillaSampler(Sampler):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        if self.args.sampler_engine == 'pt':
+            from swift.llm import PtEngine
+            _Engine = PtEngine
+        elif self.args.sampler_engine == 'vllm':
+            from swift.llm import VllmEngine
+            _Engine = VllmEngine
+        elif self.args.sampler_engine == 'lmdeploy':
+            from swift.llm import LmdeployEngine
+            _Engine = LmdeployEngine
+        elif self.args.sampler_engine == 'no':
+            _Engine = None
+        else:
+            raise ValueError(f'Cannot find engine name: {self.args.sampler_engine}')
+        self.infer_engine = None
+        if _Engine:
+            self.infer_engine = _Engine(self.args.model, model_type=self.args.model_type, **self.args.engine_kwargs)
+            self.infer_engine.default_template = self.template
+            self.infer_engine.strict = False
+        self.caches = self.read_cache()
+
+    def read_cache(self):
+        cache_files = self.args.cache_files
+        caches = {}
+        for file in cache_files:
+            if not os.path.exists(file):
+                logger.warning(f'Cache file does not exist: {file}')
+                continue
+            with open(file, 'r') as f:
+                for line in f.readlines():
+                    line = line.strip()
+                    if not line:
+                        continue
+
+                    content = json.loads(line)
+                    uuid = content['id']
+                    messages = content['messages']
+                    if uuid not in caches:
+                        caches[uuid] = {'choices': []}
+                    assert messages[-1]['role'] == 'assistant'
+                    caches[uuid]['choices'].append(messages[-1]['content'])
+        return caches
+
+    @staticmethod
+    def convert_data_to_rows(data):
+        rows = []
+        key = list(data.keys())[0]
+        data_len = len(data[key])
+        for idx in range(data_len):
+            row = {key: data[key][idx] for key in data}
+            if row.get('images') and 'bytes' in row['images'][0]:
+                row['images'] = [img['path'] for img in row['images']]
+            rows.append(row)
+        VanillaSampler.check_row_valid(rows)
+        return rows
+
+    @staticmethod
+    def check_row_valid(rows):
+        for row in rows:
+            assert not row.get('images') or all([isinstance(img, str) and img for img in row['images']])
+            assert not row.get('videos') or all([isinstance(video, str) and video for video in row['videos']])
+            assert not row.get('audios') or all([isinstance(audio, str) and audio for audio in row['audios']])
+
+    def generate(self, data):
+        resp_all = []
+        infer_requests = []
+        sent = 0
+        rows = self.convert_data_to_rows(data)
+        for idx, row in enumerate(rows):
+            row = deepcopy(row)
+            messages = row['messages']
+            uuid = get_messages_md5(row)
+            if uuid in self.caches:
+                choices = self.caches[uuid]['choices']
+                if len(choices) == self.args.num_return_sequences:
+                    continue
+            if self.args.system:
+                if messages[0]['role'] == 'system':
+                    messages[0]['content'] = self.args.system
+                else:
+                    messages.insert(0, {'role': 'system', 'content': self.args.system})
+            if messages[-1]['role'] == 'assistant':
+                messages = messages[:-1]
+
+            row['messages'] = messages
+            infer_request = row
+            for i in range(self.args.num_return_sequences):
+                infer_requests.append(deepcopy(infer_request))
+            sent += 1
+
+        request_config = RequestConfig(
+            max_tokens=self.args.max_new_tokens,
+            temperature=self.args.temperature,
+            top_k=self.args.top_k,
+            top_p=self.args.top_p,
+        )
+
+        resp_list = []
+        if len(infer_requests) > 0:
+            resp_list = self.infer_engine.infer(infer_requests, request_config=request_config)
+
+        _cur = 0
+        for idx, row in enumerate(rows):
+            row = deepcopy(row)
+            uuid = get_messages_md5(row)
+            if uuid in self.caches:
+                choices = self.caches[uuid]['choices']
+                if len(choices) == self.args.num_return_sequences:
+                    row['choices'] = choices
+                    resp_all.append(row)
+                    continue
+
+            resps = row
+            resps['choices'] = []
+            for j in range(self.args.num_return_sequences * _cur, self.args.num_return_sequences * (_cur + 1)):
+                if not isinstance(resp_list[j], Exception):
+                    resps['choices'].append(resp_list[j].choices[0].message.content)
+            if resps['choices']:
+                resp_all.append(resps)
+            _cur += 1
+        return resp_all
+
+    def do_sample(self, data):
+        generated = []
+        resp_all = self.generate(data)
+        for i, resps in enumerate(resp_all):
+            choices = resps['choices']
+            messages = resps['messages']
+            uuid = get_messages_md5(resps)
+            assert messages[-1]['role'] == 'assistant'
+            ground_truth = messages[-1]['content']
+
+            infer_requests = []
+            for decoded in choices:
+                _resps = deepcopy(resps)
+                _resps['messages'][-1]['content'] = decoded
+                infer_requests.append(_resps)
+
+            _resps = deepcopy(resps)
+            _resps['messages'][-1]['content'] = ground_truth
+            infer_requests.append(_resps)
+            if self.orm_model is not None:
+                orm_score, _orm_mask = get_reward(
+                    self.orm_model, infer_requests, ground_truths=[ground_truth] * len(infer_requests), threshold=0.0)
+            else:
+                orm_score = np.array([1.0] * len(infer_requests))
+                _orm_mask = np.array([True] * len(infer_requests))
+            if self.prm_model is not None:
+                prm_score, _prm_mask = get_reward(
+                    self.prm_model,
+                    infer_requests,
+                    ground_truths=[ground_truth] * len(infer_requests),
+                    threshold=self.args.prm_threshold)
+            else:
+                prm_score = np.array([1.0] * len(infer_requests))
+                _prm_mask = np.array([True] * len(infer_requests))
+
+            _mask = _orm_mask & _prm_mask
+            if not any(_mask):
+                continue
+
+            choices.append(ground_truth)
+            choices = np.array(choices)
+
+            if self.orm_model is None and self.prm_model is None:
+                positives = choices[:-1]
+                for positive in positives:
+                    _resps = deepcopy(resps)
+                    _resps.pop('choices', None)
+                    _resps['id'] = uuid
+                    _resps['messages'][-1]['content'] = str(positive)
+                    generated.append(json.dumps(_resps, ensure_ascii=False) + '\n')
+            else:
+                score = np.array(prm_score) + np.array(orm_score * 10)
+                sorted_indices = np.argsort(score)[::-1]
+                pos_indexes = sorted_indices[0:self.args.n_best_to_keep]
+                pos_indexes = [i for i in pos_indexes if _mask[i]]
+                neg_index = sorted_indices[-1]
+                logger.info(
+                    f'orm:{orm_score}, prm:{prm_score}, positive index: {pos_indexes}, negative index: {neg_index}')
+                if self.args.easy_query_threshold is not None and sum([score > 0 for score in orm_score]) - 1 >= int(
+                        self.args.num_return_sequences * self.args.easy_query_threshold):
+                    continue
+                if len(pos_indexes) > 0:
+                    positives = choices[pos_indexes]
+                    negative = choices[neg_index]
+                    for positive in positives:
+                        _resps = deepcopy(resps)
+                        messages = deepcopy(messages)
+                        _resps.pop('choices', None)
+                        _resps['messages'][-1]['content'] = str(positive)
+                        _resps['rejected_response'] = str(negative)
+                        _resps['id'] = uuid
+                        generated.append(json.dumps(_resps, ensure_ascii=False) + '\n')
+        return generated

swift/llm/template/__init__.py

@@ -0,0 +1,10 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from . import template
+from .base import MaxLengthError, Template
+from .constant import TemplateType
+from .grounding import draw_bbox
+from .register import TEMPLATE_MAPPING, get_template, get_template_meta, register_template
+from .template_inputs import InferRequest, TemplateInputs
+from .template_meta import TemplateMeta
+from .utils import Prompt, Word, split_str_parts_by
+from .vision_utils import load_file, load_image

swift/llm/template/base.py

@@ -0,0 +1,1606 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import hashlib
+import inspect
+import math
+import os
+import re
+from contextlib import contextmanager, nullcontext
+from copy import deepcopy
+from dataclasses import asdict
+from functools import partial, wraps
+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from modelscope.hub.utils.utils import get_cache_dir
+from peft import PeftModel
+from PIL import Image
+from torch.nn.utils.rnn import pad_sequence
+from transformers import StoppingCriteriaList
+from transformers.integrations import is_deepspeed_zero3_enabled
+from transformers.utils import strtobool
+
+from swift.utils import get_dist_setting, get_env_args, get_logger, use_torchacc
+from ..utils import Processor, ProcessorMixin
+from .template_inputs import InferRequest, StdTemplateInputs, TemplateInputs
+from .utils import Context, ContextType, StopWordsCriteria, fetch_one, findall, split_str_parts_by
+from .vision_utils import load_audio, load_batch, load_image, rescale_image
+
+logger = get_logger()
+if TYPE_CHECKING:
+    from .template_meta import TemplateMeta
+
+
+class MaxLengthError(ValueError):
+    pass
+
+
+class Template(ProcessorMixin):
+    special_tokens = ['<image>', '<video>', '<audio>', '<bbox>', '<ref-object>', '<cot-process>', '<start-image>']
+    special_keys = ['images', 'videos', 'audios', 'objects']
+
+    image_placeholder = ['<image>']
+    video_placeholder = ['<video>']
+    audio_placeholder = ['<audio>']
+    cot_process_placeholder = ['ки']
+    placeholder_tokens = []  # For clearer printing
+    load_images = True
+    skip_prompt = True
+    use_model = False
+    norm_bbox = 'norm1000'
+
+    is_encoder_decoder = False
+
+    def __init__(
+        self,
+        processor: Processor,
+        template_meta: 'TemplateMeta',
+        default_system: Optional[str] = None,
+        max_length: Optional[int] = None,
+        *,
+        use_chat_template: bool = True,
+        truncation_strategy: Literal['raise', 'left', 'right'] = 'raise',
+        max_pixels: Optional[int] = None,
+        agent_template: Optional[str] = None,
+        norm_bbox: Literal['norm1000', 'none', None] = None,
+        response_prefix: Optional[str] = None,
+        # only for train
+        padding_side: Literal['left', 'right'] = 'right',
+        loss_scale: str = 'default',
+        sequence_parallel_size: int = 1,
+        # infer/deploy
+        template_backend: Literal['swift', 'jinja'] = 'swift',
+    ) -> None:
+        """
+        default_system: Override the default_system in the template.
+        max_length: Max length of the sequence
+        truncation_strategy: The truncation strategy
+        max_pixels: Rescale image to reduce memory usage, default `None` means no limitation.
+            e.g. 512 * 512 (H*W)
+        padding_side: The padding_side when the training batch_size >= 2
+        loss_scale: The loss scale function to use
+        """
+        from .template_meta import TemplateMeta
+        from swift.plugin import agent_templates
+
+        self.processor = processor
+        self.model_info = processor.model_info
+        self.config = self.model_info.config
+        self.model_meta = processor.model_meta
+        if max_length is None:
+            max_length = self.model_info.max_model_len
+        tokenizer = self.tokenizer
+
+        if not use_chat_template:
+            template_meta = template_meta.to_generate_template_meta()
+        else:
+            template_meta = deepcopy(template_meta)
+        # if default_system is None. not change self.default_system
+        template_meta.check_system(default_system)
+        if default_system is not None:
+            template_meta.default_system = default_system
+        if response_prefix is not None:
+            template_meta.response_prefix = response_prefix
+        logger.info(f'default_system: {repr(template_meta.default_system)}')
+        logger.info(f'response_prefix: {repr(template_meta.response_prefix)}')
+
+        for i, token in enumerate(self.placeholder_tokens):
+            if isinstance(token, str):
+                self.placeholder_tokens[i] = tokenizer.convert_tokens_to_ids(token)
+        template_meta.init(tokenizer)
+
+        self.template_meta: TemplateMeta = template_meta
+        self.use_chat_template = use_chat_template
+        self.template_backend = template_backend
+        self.max_length = max_length
+        self.truncation_strategy = truncation_strategy
+        self.loss_scale = loss_scale
+        self.max_pixels = max_pixels
+        self.padding_side = padding_side
+        self.sequence_parallel_size = sequence_parallel_size
+        agent_template = agent_template or template_meta.agent_template
+        logger.info(f'agent_template: {agent_template}')
+        self.agent_template = agent_templates[agent_template]()
+        self.norm_bbox = norm_bbox or self.norm_bbox
+        logger.info(f'max_length: {self.max_length}')
+        logger.info(f'norm_bbox: {self.norm_bbox}')
+        if self.is_encoder_decoder:
+            self.skip_prompt = False
+        self.mode: Literal['pt', 'vllm', 'lmdeploy',  # infer
+                           'train', 'rlhf', 'kto',  # train
+                           'seq_cls', 'embedding', 'prm'] = 'pt'
+        self._packing = False
+        self.use_megatron = False
+        if self.model_info.task_type != 'causal_lm':
+            self.mode = self.model_info.task_type
+        self._handles = []
+        self._deepspeed_initialize = None
+
+    @staticmethod
+    def _load_image(image, load_images: bool):
+        if load_images:
+            if isinstance(image, dict) and 'bytes' in image:
+                image = image['bytes'] or image['path']
+            image = load_image(image)
+        else:
+            if isinstance(image, dict):
+                path = image['path']
+                if path and (path.startswith('http') or os.path.exists(path)):
+                    image = path
+                else:
+                    image = load_image(image['bytes'])
+            elif not isinstance(image, str):
+                image = load_image(image)
+        return image
+
+    @staticmethod
+    def _get_height_width(inputs: StdTemplateInputs) -> None:
+        width = []
+        height = []
+        for image in inputs.images:
+            width.append(image.width)
+            height.append(image.height)
+        inputs.objects['width'] = width
+        inputs.objects['height'] = height
+
+    def normalize_bbox(self, inputs: StdTemplateInputs) -> None:
+        objects = inputs.objects
+        bbox_list = objects['bbox']
+        width_list = objects['width']
+        height_list = objects['height']
+        bbox_type = objects.pop('bbox_type', None) or 'real'
+        image_id_list = objects.pop('image_id', None) or []
+        image_id_list += [0] * (len(bbox_list) - len(image_id_list))
+        for bbox, image_id in zip(bbox_list, image_id_list):
+            if bbox_type == 'norm1':
+                width, height = 1, 1
+            else:
+                width, height = width_list[image_id], height_list[image_id]
+            for i, (x, y) in enumerate(zip(bbox[::2], bbox[1::2])):
+                if self.norm_bbox == 'norm1000':
+                    norm_width, norm_height = 1000, 1000
+                elif self.norm_bbox == 'none':
+                    image = inputs.images[image_id]
+                    norm_width, norm_height = image.width, image.height
+                bbox[2 * i] = int(round(x / width * norm_width))
+                bbox[2 * i + 1] = int(round(y / height * norm_height))
+
+    def _preprocess_function_call(self, inputs: StdTemplateInputs) -> None:
+        agent_template = self.agent_template
+        agent_template.template_meta = self.template_meta  # for hermes
+        if inputs.tools:
+            if isinstance(inputs.tools, str):
+                inputs.tools = agent_template._parse_json(inputs.tools)
+                if not isinstance(inputs.tools, (list, tuple)):
+                    inputs.tools = [inputs.tools]
+            elif isinstance(inputs.tools, (list, tuple)):
+                inputs.tools = [agent_template._parse_json(tool) for tool in inputs.tools]
+            else:
+                raise ValueError(f'inputs.tools: {inputs.tools}')
+            for i, tool in enumerate(inputs.tools):
+                inputs.tools[i] = agent_template.wrap_tool(tool)
+        i = 0
+        messages = inputs.messages
+        while i < len(messages):
+            if messages[i]['role'] == 'tool_call':
+                i_start = i
+                while i + 1 < len(messages) and messages[i + 1]['role'] == 'tool_call':
+                    i += 1
+                tool_content = self.agent_template._format_tool_calls(messages[i_start:i + 1])
+                messages[i_start:i + 1] = [{'role': 'assistant', 'content': tool_content}]
+                i = i_start + 1
+            else:
+                i += 1
+
+    def _preprocess_inputs(
+        self,
+        inputs: StdTemplateInputs,
+    ) -> None:
+        self._preprocess_function_call(inputs)
+        if self.model_meta.is_multimodal:
+            self._replace_image_tags(inputs)
+            self._replace_start_image_tags(inputs)
+        images = inputs.images
+        load_images = self.load_images or self.mode in {'vllm', 'lmdeploy'}
+        load_images_origin = load_images
+        if self.max_pixels is not None or inputs.objects:
+            load_images = True
+        if images:
+            for i, image in enumerate(images):
+                images[i] = self._load_image(images[i], load_images)
+        if inputs.objects:
+            self._get_height_width(inputs)
+        if self.max_pixels is not None:
+            # Scale the image proportionally without affecting the scaled objects.
+            images = [rescale_image(img, self.max_pixels) for img in images]
+        if images and not load_images_origin:  # fix pt & qwen-vl
+            for i, image in enumerate(images):
+                if isinstance(image, Image.Image):
+                    images[i] = self._save_pil_image(image)
+        inputs.images = images
+
+        if self.mode == 'vllm' and inputs.audios:
+            sampling_rate = get_env_args('sampling_rate', int, None)
+            inputs.audios = load_batch(
+                inputs.audios, load_func=partial(load_audio, sampling_rate=sampling_rate, return_sr=True))
+
+        if inputs.is_multimodal:
+            self._add_default_tags(inputs)
+
+    @staticmethod
+    def _replace_image_tags(inputs: StdTemplateInputs):
+        # compat
+        if inputs.images:
+            return
+        images = []
+        pattern = r'<img>(.+?)</img>'
+        for message in inputs.messages:
+            content = message['content']
+            if not isinstance(content, str):
+                continue
+            for image in re.findall(pattern, content):
+                # only support local_path
+                if os.path.isfile(image):
+                    images.append(image)
+                else:
+                    logger.warning_once(f'Failed to parse image path: `{content}`.', hash_id='<img></img>')
+            message['content'] = re.sub(pattern, '<image>', content)
+        inputs.images = images
+
+    @staticmethod
+    def _replace_start_image_tags(inputs: StdTemplateInputs):
+        # compat
+        generate_mode = False
+        message = inputs.messages[-1]
+        content = message['content']
+        if message['role'] == 'user' and content.endswith('<start-image>'):
+            generate_mode = True
+            message['content'] = message['content'][:-len('<start-image>')]  # remove the <start-image>
+        inputs.generate_mode = generate_mode
+
+    @staticmethod
+    def _extend_tokens(input_ids: List[int], labels: Optional[List[int]], replace_idx_list: List[int],
+                       get_new_tokens: Callable[[int], List[int]]) -> Tuple[List[int], Optional[List[int]]]:
+        added_tokens_len = 0
+        for i, idx in enumerate(replace_idx_list):
+            new_tokens = get_new_tokens(i)
+            token_len = len(new_tokens)
+            input_ids = input_ids[:idx + added_tokens_len] + new_tokens + input_ids[added_tokens_len + idx + 1:]
+            if labels:
+                labels = labels[:idx + added_tokens_len] + [-100] * token_len + labels[added_tokens_len + idx + 1:]
+            added_tokens_len += token_len - 1
+        return input_ids, labels
+
+    def compute_loss_context(self, model, inputs):
+        return nullcontext()
+
+    def _rlhf_encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        chosen_inputs, rejected_inputs = inputs, deepcopy(inputs)
+        assert chosen_inputs.rejected_response is not None, f'inputs: {inputs}'
+        rejected_inputs.messages[-1]['content'] = chosen_inputs.rejected_response
+        chosen_encoded = self._encode_truncated(chosen_inputs)
+        rejected_encoded = self._encode_truncated(rejected_inputs)
+
+        encoded = {}
+        for prefix in ['chosen', 'rejected']:
+            data = locals()[f'{prefix}_encoded']
+            for k, v in data.items():
+                encoded[f'{prefix}_{k}'] = v
+        return encoded
+
+    def _kto_encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        label, inputs.label = inputs.label, None
+        encoded = self._rlhf_encode(inputs)
+        encoded['label'] = bool(label)
+        return encoded
+
+    def _embedding_encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        _encoded = {}
+        labels = []
+
+        def split_multi_medias(_inputs):
+            _content = _inputs.messages[-2]['content']
+            image_size = len(re.findall('<image>', _content))
+            video_size = len(re.findall('<video>', _content))
+            audio_size = len(re.findall('<audio>', _content))
+            _inputs.images = inputs.images[:image_size]
+            assert len(_inputs.images) == image_size
+            inputs.images = inputs.images[image_size:]
+            _inputs.videos = inputs.videos[:video_size]
+            assert len(_inputs.videos) == video_size
+            inputs.videos = inputs.videos[video_size:]
+            _inputs.audios = inputs.audios[:audio_size]
+            assert len(_inputs.audios) == audio_size
+            inputs.audios = inputs.audios[audio_size:]
+
+        anchor = deepcopy(inputs)
+        anchor.messages[-1]['content'] = ''
+        anchor.rejected_response = []
+        split_multi_medias(anchor)
+        anchor_encoded = self._encode_truncated(anchor)
+        for key in anchor_encoded:
+            _encoded[f'anchor_{key}'] = anchor_encoded[key]
+
+        positive = deepcopy(inputs)
+        positive.messages[-2]['content'] = positive.messages[-1]['content']
+        positive.messages[-1]['content'] = ''
+        positive.rejected_response = []
+        split_multi_medias(positive)
+        positive_encoded = self._encode_truncated(positive)
+        for key in positive_encoded:
+            _encoded[f'positive_{key}'] = positive_encoded[key]
+        labels.append(float(inputs.label) if inputs.label is not None else 1.0)
+
+        rejected_len = len(inputs.rejected_response) if inputs.rejected_response else 0
+        for i in range(rejected_len):
+            negative = deepcopy(inputs)
+            negative.messages[-2]['content'] = negative.rejected_response[i]
+            negative.messages[-1]['content'] = ''
+            negative.rejected_response = []
+            split_multi_medias(negative)
+            negative_encoded = self._encode_truncated(negative)
+            for key in negative_encoded:
+                _encoded[f'negative{i}_{key}'] = negative_encoded[key]
+            labels.append(0.0)
+
+        _encoded['labels'] = labels
+        return _encoded
+
+    def _seq_cls_encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = self._encode_truncated(inputs)
+        encoded.pop('labels', None)
+        if inputs.label is not None:
+            labels = inputs.label
+            problem_type = self._get_problem_type(self.config, labels=labels)
+            if problem_type == 'single_label_classification':
+                labels = int(labels)
+            encoded['labels'] = labels
+        return encoded
+
+    @torch.inference_mode()
+    def encode(self,
+               inputs: Union[TemplateInputs, Dict[str, Any], InferRequest],
+               return_template_inputs: bool = False) -> Dict[str, Any]:
+        """The entrance method of Template!
+
+        Returns:
+            return {'input_ids': List[int], 'labels': Optional[List[int]], ...}
+        """
+        if isinstance(inputs, (InferRequest, TemplateInputs)):
+            inputs = asdict(inputs)
+
+        if isinstance(inputs, dict):
+            inputs = deepcopy(inputs)
+            if not self.is_training:
+                InferRequest.remove_response(inputs['messages'])
+            inputs = StdTemplateInputs.from_dict(inputs)
+        elif isinstance(inputs, StdTemplateInputs):
+            inputs = deepcopy(inputs)
+        assert isinstance(inputs, StdTemplateInputs)
+        self._preprocess_inputs(inputs)
+
+        if self.mode in {'pt', 'train', 'prm', 'vllm', 'lmdeploy'}:
+            encoded = self._encode_truncated(inputs)
+        elif self.mode == 'seq_cls':
+            encoded = self._seq_cls_encode(inputs)
+        elif self.mode == 'rlhf':
+            encoded = self._rlhf_encode(inputs)
+        elif self.mode == 'kto':
+            encoded = self._kto_encode(inputs)
+        elif self.mode == 'embedding':
+            encoded = self._embedding_encode(inputs)
+        for key in list(encoded.keys()):
+            if encoded[key] is None:
+                encoded.pop(key)
+        if return_template_inputs:
+            encoded['template_inputs'] = inputs
+        return encoded
+
+    def packing_row(self, row: List[Tuple[Dict[str, Any], int]]) -> Dict[str, Any]:
+        packed = {}
+        keys = set()
+        for r in row:
+            keys.update(r[0].keys())
+        for key in keys:
+            if key in {'input_ids', 'labels', 'loss_scale'}:
+                packed[key] = sum((x[0][key] for x in row), start=[])
+        if 'position_ids' not in packed:
+            packed['position_ids'] = sum((list(range(x[1])) for x in row), start=[])
+
+        packed.update(self._data_collator_mm_data([r[0] for r in row]))
+        return packed
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        return inputs
+
+    @staticmethod
+    def _skip_stop_tokens(generate_ids: List[int], stop_tokens: List[int], is_finished: bool) -> List[int]:
+        len_tokens = len(stop_tokens)
+        if is_finished and generate_ids[-len_tokens:] == stop_tokens:
+            return generate_ids[:-len_tokens]
+        if not is_finished:
+            for i in range(len_tokens, 0, -1):
+                if generate_ids[-i:] == stop_tokens[:i]:
+                    return generate_ids[:-i]
+        return generate_ids
+
+    @staticmethod
+    def _get_seq_cls_logprobs(pred: int, logprobs: torch.Tensor, top_logprobs: int):
+        idxs = logprobs.argsort(descending=True, dim=-1)[:top_logprobs].tolist()
+        logprobs = logprobs.tolist()
+        return {
+            'content': [{
+                'index': pred,
+                'logprobs': [logprobs[p] for p in pred] if isinstance(pred, (list, tuple)) else logprobs[pred],
+                'top_logprobs': [{
+                    'index': idx,
+                    'logprob': logprobs[idx]
+                } for idx in idxs]
+            }]
+        }
+
+    @staticmethod
+    def _get_problem_type(config, labels=None, logits=None) -> str:
+        problem_type = config.problem_type
+        if problem_type is not None:
+            return problem_type
+        if labels is not None:
+            if isinstance(labels, (list, tuple)):
+                if labels and isinstance(labels[0], float):
+                    problem_type = 'regression'
+                else:
+                    problem_type = 'multi_label_classification'
+            else:
+                problem_type = 'single_label_classification'
+                assert config.num_labels >= labels + 1
+        if logits is not None:
+            if logits.shape[-1] == 1:
+                problem_type = 'regression'
+            else:
+                problem_type = 'single_label_classification'  # compatible with older versions
+        assert problem_type is not None
+        config.problem_type = problem_type
+        return problem_type
+
+    def decode_seq_cls(self, logits: torch.Tensor, top_logprobs: int):
+        assert isinstance(logits, torch.Tensor)
+        problem_type = self._get_problem_type(self.config, logits=logits)
+        if problem_type == 'regression':
+            preds = logits.squeeze(dim=-1).tolist()
+            logprobs = [None] * len(preds)
+        else:
+            if problem_type == 'single_label_classification':
+                preds = torch.argmax(logits, dim=-1).tolist()
+                logprobs = torch.log_softmax(logits, -1)
+            else:
+                preds = [(logprob >= 0.5).nonzero(as_tuple=True)[0].tolist() for logprob in torch.sigmoid(logits)]
+                logprobs = F.logsigmoid(logits)
+            logprobs = [self._get_seq_cls_logprobs(pred, logprobs[i], top_logprobs) for i, pred in enumerate(preds)]
+        return preds, logprobs
+
+    def decode(self,
+               generate_ids: List[int],
+               *,
+               is_finished: bool = True,
+               tokenizer_kwargs=None,
+               first_token=True,
+               **kwargs) -> Any:
+        tokenizer_kwargs = tokenizer_kwargs or {}
+        response = self._skip_stop_decode(generate_ids, is_finished, **tokenizer_kwargs)
+        if first_token and self.template_meta.response_prefix:
+            response = self.template_meta.response_prefix + response
+        return response
+
+    def decode_prm(self, input_ids: torch.Tensor, logits: torch.Tensor) -> Any:
+        raise NotImplementedError
+
+    def generate(self, model, *args, **kwargs):
+        if isinstance(model, PeftModel):
+            signature = inspect.signature(model.model.generate)
+        else:
+            signature = inspect.signature(model.generate)
+        if 'use_model_defaults' in signature.parameters and 'use_model_defaults' not in kwargs:
+            kwargs['use_model_defaults'] = False
+        return model.generate(*args, **kwargs)
+
+    def _skip_stop_decode(self, generate_ids: List[int], is_finished: bool, **decode_kwargs) -> Any:
+        # Do not print template_meta.suffix[-1] and eos_token.
+        # However, other stop_words will be printed.
+        tokenizer = self.tokenizer
+
+        if len(generate_ids) > 0 and generate_ids[-1] == tokenizer.eos_token_id:
+            generate_ids = generate_ids[:-1]
+        # skip suffix and eos_token
+        template_suffix = self.template_meta.suffix[-1]
+        if isinstance(template_suffix, str):
+            # [-1:]: fix OpenGVLab/Mini-InternVL-Chat-4B-V1-5
+            template_suffix = tokenizer.encode(template_suffix, add_special_tokens=False)[-1:]
+        generate_ids = self._skip_stop_tokens(generate_ids, template_suffix, is_finished)
+        if 'spaces_between_special_tokens' not in decode_kwargs:
+            decode_kwargs['spaces_between_special_tokens'] = False
+        return tokenizer.decode(generate_ids, **decode_kwargs)
+
+    def prepare_generate_kwargs(self, generate_kwargs: Dict[str, Any], *, model=None) -> Dict[str, Any]:
+        generation_config = generate_kwargs['generation_config']
+        stop_words = getattr(generation_config, 'stop_words', None) or self.template_meta.stop_words
+        generate_kwargs['stopping_criteria'] = StoppingCriteriaList([StopWordsCriteria(self.tokenizer, stop_words)])
+        return generate_kwargs
+
+    @staticmethod
+    def _save_pil_image(image: Image.Image) -> str:
+        img_bytes = image.tobytes()
+        img_hash = hashlib.sha256(img_bytes).hexdigest()
+        tmp_dir = os.path.join(get_cache_dir(), 'tmp', 'images')
+        logger.info_once(f'create tmp_dir: {tmp_dir}')
+        os.makedirs(tmp_dir, exist_ok=True)
+        img_path = os.path.join(tmp_dir, f'{img_hash}.png')
+        if not os.path.exists(img_path):
+            image.save(img_path)
+        return img_path
+
+    @staticmethod
+    def _concat_context_list(
+            context_list: List[Context],
+            res_context_list: List[Context],  # inplace
+            res_context_type: List[ContextType],  # inplace
+            system: Optional[str] = None,
+            query: Optional[str] = None,
+            response: Optional[str] = None,
+            round0: Optional[int] = None) -> None:
+        """Concat context list and replace placeholder"""
+        round1 = None
+        if round0 is not None:
+            round1 = str(round0 + 1)
+            round0 = str(round0)
+        for context in context_list:
+            if isinstance(context, str):
+                if '{{RESPONSE}}' == context:
+                    assert response is not None
+                    res_context_list.append(response)
+                    res_context_type.append(ContextType.RESPONSE)
+                    continue
+                old_str_list = ['{{SYSTEM}}', '{{QUERY}}', '{{ROUND0}}', '{{ROUND1}}']
+                new_str_list = [system, query, round0, round1]
+                for (old_str, new_str) in zip(old_str_list, new_str_list):
+                    if new_str is not None and old_str in context:
+                        assert isinstance(new_str, str), f'new_str: {new_str}'
+                        context = context.replace(old_str, new_str)
+            if len(context) == 0:
+                continue
+            res_context_list.append(context)
+            res_context_type.append(ContextType.OTHER)
+
+    def _simplify_context_list(self, context_list: List[Context], loss_scale_list: List[float],
+                               inputs: StdTemplateInputs) -> Tuple[List[Context], List[float]]:
+        """Merge anything in the context to simplify the inputs"""
+        context_list, loss_scale_list = self._split_special_tokens(context_list, loss_scale_list)
+        context_list, loss_scale_list = self._pre_tokenize(context_list, loss_scale_list, inputs)
+
+        res: List[Context] = []  # result of context_list
+        res_loss_scale: List[float] = []  # result of loss_scale_list
+        temp: List[str] = []
+        temp_loss_scale = 0.
+        for i, (context, loss_scale) in enumerate(zip(context_list, loss_scale_list)):
+            if isinstance(context, str) and (loss_scale == temp_loss_scale):
+                temp.append(context)
+            else:
+                if len(temp) > 0:
+                    res.append(''.join(temp))
+                    res_loss_scale.append(temp_loss_scale)
+                    temp.clear()
+                if isinstance(context, str):  # loss_scale diff
+                    temp.append(context)
+                else:
+                    res.append(context)
+                    res_loss_scale.append(loss_scale)
+                temp_loss_scale = loss_scale
+        if len(temp) > 0:
+            res.append(''.join(temp))
+            res_loss_scale.append(temp_loss_scale)
+
+        return res, res_loss_scale
+
+    @staticmethod
+    def _split_special_tokens(context_list: List[Context],
+                              loss_scale_list: List[float]) -> Tuple[List[Context], List[float]]:
+        """Split special tokens, for example `<image>`, `<video>`, this will help the replace_tag operation"""
+        res: List[Context] = []
+        loss_scale_res: List[float] = []
+        for context, loss_scale in zip(context_list, loss_scale_list):
+            contexts = []
+            if isinstance(fetch_one(context), str):
+                for d in split_str_parts_by(context, Template.special_tokens):
+                    contexts.extend([d['key'], d['content']])
+                contexts = [c for c in contexts if c]
+                res.extend(contexts)
+                loss_scale_res.extend([loss_scale] * len(contexts))
+            else:
+                res.append(context)
+                loss_scale_res.append(loss_scale)
+        return res, loss_scale_res
+
+    def _tokenize(self, context, **tokenizer_kwargs):
+        return self.tokenizer(
+            context, return_attention_mask=False, add_special_tokens=False, **tokenizer_kwargs)['input_ids']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        """Override this function to do your own replace operation.
+
+        This method is used to replace standard tags like `<image>` to some tokens that the model needs.
+
+        Args:
+            media_type: The modal.
+            index: The index of the medias, for index 0 represents the first elements in `images`
+            inputs: The inputs
+
+        Returns:
+            The content or input_ids after replacement.
+        """
+        if media_type == 'image':
+            if self.mode == 'lmdeploy':
+                return [[-100]]
+            return self.image_placeholder
+        elif media_type == 'video':
+            return self.video_placeholder
+        elif media_type == 'audio':
+            return self.audio_placeholder
+
+    def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:
+        """Replace objects referenced by the bbox to contents or input_ids. This is useful in the grounding task.
+        Override this function to do your own replace operation.
+
+        Args:
+            ref: Description of the bbox
+            index: The index in the `objects` key
+            inputs: The inputs
+
+        Returns:
+            The contents or input_ids replaced
+        """
+        return [ref]
+
+    def replace_cot_process(self, inputs: StdTemplateInputs) -> List[Context]:
+        """Replace the cot process label for PRM training or inference.
+        Override this function to do your own replace operation.
+
+        Args:
+            inputs: The inputs
+
+        Returns:
+            The contents or input_ids replaced
+        """
+        return [self.cot_process_placeholder]
+
+    @staticmethod
+    def _get_bbox_str(bbox: List[int]) -> str:
+        point = []
+        for x, y in zip(bbox[::2], bbox[1::2]):
+            point.append(f'({x},{y})')
+        return ','.join(point)
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        """Replace bbox pointing to the objects to contents or input_ids. This is useful in the grounding task.
+        Override this function to do your own replace operation.
+
+        Args:
+            bbox: [x, y] or [x1, y1, x2, y2]
+            index: The index in the `objects` key
+            inputs: The inputs
+
+        Returns:
+            The contents or input_ids replaced
+        """
+        return [f'[{self._get_bbox_str(bbox)}]']
+
+    def _pre_tokenize_images(self, context_list: List[Context], loss_scale_list: List[float],
+                             inputs: StdTemplateInputs) -> Tuple[List[Context], List[float]]:
+        # https://github.com/modelscope/ms-swift/issues/3407
+        # Fix the bounding box position offset issue in the Qwen2.5-VL grounding task.
+        res: List[Context] = []
+        res_loss_scale: List[float] = []
+        inputs.image_idx = 0
+
+        for context, loss_scale in zip(context_list, loss_scale_list):
+            if context == '<image>' and inputs.is_multimodal and inputs.image_idx < len(inputs.images):
+                c_list = self.replace_tag('image', inputs.image_idx, inputs)
+                inputs.image_idx += 1
+                loss_scale = 0. if self.template_backend == 'swift' else 1.
+            else:
+                c_list = [context]
+            res += c_list
+            res_loss_scale += [loss_scale] * len(c_list)
+        return res, res_loss_scale
+
+    def _pre_tokenize(self, context_list: List[Context], loss_scale_list: List[float],
+                      inputs: StdTemplateInputs) -> Tuple[List[Context], List[float]]:
+        """This method happens before tokenization, replace standard tags to the contents or input_ids needed by
+        the model.
+
+        Args:
+            context_list: The content list
+            loss_scale_list: The loss scale list
+        Returns:
+            The context_list and loss_scale_list after replacement.
+        """
+        context_list, loss_scale_list = self._pre_tokenize_images(context_list, loss_scale_list, inputs)
+        if inputs.images and inputs.objects:
+            self.normalize_bbox(inputs)
+        # replace tag/object/box
+        res: List[Context] = []  # result of context_list
+        res_loss_scale: List[float] = []  # result of loss_scale_list
+
+        # reset
+        for k in ['video', 'audio', 'object', 'box']:
+            setattr(inputs, f'{k}_idx', 0)
+
+        for context, loss_scale in zip(context_list, loss_scale_list):
+            for k in ['video', 'audio']:
+                if context == f'<{k}>' and inputs.is_multimodal and getattr(inputs, f'{k}_idx') < len(
+                        getattr(inputs, f'{k}s')):
+                    c_list = self.replace_tag(k, getattr(inputs, f'{k}_idx'), inputs)
+                    setattr(inputs, f'{k}_idx', getattr(inputs, f'{k}_idx') + 1)
+                    loss_scale = 0.
+                    break
+            else:
+                ref = inputs.objects.get('ref') or []
+                bbox = inputs.objects.get('bbox') or []
+                if context == '<ref-object>' and inputs.ref_idx < len(ref):
+                    idx = inputs.ref_idx
+                    c_list = self.replace_ref(ref[idx], idx, inputs)
+                    inputs.ref_idx += 1
+                elif context == '<bbox>' and inputs.bbox_idx < len(bbox):
+                    idx = inputs.bbox_idx
+                    c_list = self.replace_bbox(bbox[idx], idx, inputs)
+                    inputs.bbox_idx += 1
+                elif context == '<cot-process>' and self.mode == 'prm':
+                    c_list = self.replace_cot_process(inputs)
+                else:
+                    c_list = [context]
+            res += c_list
+            res_loss_scale += [loss_scale] * len(c_list)
+        return res, res_loss_scale
+
+    @staticmethod
+    def _add_default_tags(inputs: StdTemplateInputs):
+        total_content = '\n'.join([message['content'] or '' for message in inputs.messages])
+        if inputs.rejected_response:
+            if isinstance(inputs.rejected_response, str):
+                total_content += inputs.rejected_response
+            else:
+                total_content += '\n'.join(inputs.rejected_response)
+        if inputs.system:
+            total_content = f'{inputs.system}\n{total_content}'
+        for media_type in ['image', 'audio', 'video']:
+            media_key, media_tag = f'{media_type}s', f'<{media_type}>'
+            medias = getattr(inputs, media_key)
+            if not isinstance(medias, list):
+                medias = [medias]
+            if medias:
+                num_media_tags = len(re.findall(media_tag, total_content))
+                num_media = len(medias)
+                num_new_tags = num_media - num_media_tags
+                if num_new_tags > 0:
+                    inputs.messages[0]['content'] = media_tag * num_new_tags + inputs.messages[0]['content']
+                elif num_new_tags < 0:
+                    logger.warning(
+                        f'num_media: {num_media}, num_media_tags: {num_media_tags}, total_content: {total_content}. '
+                        'We will only replace the frontmost media_tags while keeping the subsequent media_tags.')
+
+    def _encode_context_list(
+            self,
+            context_list: List[Context],
+            loss_scale_list: Optional[List[float]] = None) -> Tuple[List[int], List[int], List[float], Dict[str, Any]]:
+        """return: input_ids, labels, tokenizer_kwargs"""
+        input_ids: List[int] = []
+        labels: List[int] = []
+        loss_scale: List[float] = []
+        tokenizer_kwargs = {}
+        if loss_scale_list is None:
+            loss_scale_list = [0.] * len(context_list)
+        ignore_loss_scale = all(loss_scale in {0, 1} for loss_scale in loss_scale_list)
+        for i, (context, loss_weight) in enumerate(zip(context_list, loss_scale_list)):
+            if isinstance(context, str):
+                # tokenizer_kwargs is the returned tokenizer_kwargs,
+                # while curr_tokenizer_kwargs is the tokenizer_kwargs for the current context.
+                token_list = self._tokenize(context)
+            else:
+                token_list = context
+            input_ids += token_list
+            if loss_scale_list[i] > 0.0:
+                labels += token_list
+            else:
+                labels += [-100] * len(token_list)
+            if not ignore_loss_scale:
+                loss_scale.extend([loss_weight] * len(token_list))
+        if ignore_loss_scale:
+            loss_scale = None
+        return input_ids, labels, loss_scale, tokenizer_kwargs
+
+    @staticmethod
+    def _add_dynamic_eos(input_ids: List[int], labels: List[int], loss_scale: Optional[List[int]],
+                         suffix_tokens_id: List[int]) -> None:
+        suffix_len = len(suffix_tokens_id)
+        start = 0
+        for i in range(1, len(labels)):
+            if labels[i - 1] >= 0 and labels[i] == -100:
+                start = i
+            if start > 0 and labels[i - 1] == -100 and labels[i] >= 0:
+                # [0, 1, 2, -100(start), -100, 3(i), 4]
+                length = i - start
+                if length >= suffix_len and input_ids[start:start + suffix_len] == suffix_tokens_id:
+                    labels[start:start + suffix_len] = suffix_tokens_id
+                    if loss_scale and loss_scale[start:start + suffix_len] == [0] * suffix_len:
+                        loss_scale[start:start + suffix_len] = [1] * suffix_len
+
+    @staticmethod
+    def _get_std_messages(messages):
+        if messages and messages[0]['role'] == 'assistant':
+            messages.insert(0, {'role': 'user', 'content': ''})  # pretrain
+        if len(messages) % 2 == 1:
+            messages.append({'role': 'assistant', 'content': None})  # inference
+
+    def _jinja_encode(self, inputs: StdTemplateInputs):
+        messages = inputs.messages.copy()
+        if inputs.system is not None:
+            messages.insert(0, {'role': 'system', 'content': inputs.system})
+        if messages[-1]['content'] is None:
+            messages.pop()
+        add_generation_prompt = messages[-1]['role'] != 'assistant'
+        kwargs = {}
+        if inputs.tools:
+            kwargs['tools'] = inputs.tools
+        text = self.tokenizer.apply_chat_template(
+            messages, tokenize=False, add_generation_prompt=add_generation_prompt, **kwargs)
+        answer_len = 1 if self.is_training else 0
+        return [text], [1.], answer_len
+
+    def _get_system(self, inputs) -> Optional[str]:
+        template_meta = self.template_meta
+        system = inputs.system
+        tools = inputs.tools
+        template_meta.check_system(system)
+        if system is None:
+            system = template_meta.default_system
+
+        if tools is not None:
+            system = self.agent_template._format_tools(tools, system or '', inputs.messages[0])
+        return system
+
+    @staticmethod
+    def _swift_prepare_function_call(agent_template, messages):
+        if len(messages) < 2:
+            return
+        i = 1
+        while i < len(messages):
+            pre_message, message = messages[i - 1], messages[i]
+            pre_role, pre_content = pre_message['role'], pre_message['content']
+            role, content = message['role'], message['content']
+            if pre_role == 'assistant' and role == 'tool':
+                i_start = i
+                while i + 1 < len(messages) and messages[i + 1]['role'] == 'tool':
+                    i += 1
+                pre_message['content'], tool_content = agent_template._format_tool_responses(
+                    pre_content, messages[i_start:i + 1])
+                messages[i_start:i + 1] = [{'role': 'tool', 'content': tool_content}]
+                i = i_start + 1
+            elif pre_role == 'assistant' and role == 'assistant':
+                # Consecutive messages from the assistant role need to be merged to prevent errors.
+                pre_message['content'] = pre_content + content
+                messages.pop(i)
+            else:
+                i += 1
+
+    def _swift_encode(self, inputs: StdTemplateInputs):
+        template_meta = self.template_meta
+        system = self._get_system(inputs)
+        self._swift_prepare_function_call(self.agent_template, inputs.messages)
+
+        self._get_std_messages(inputs.messages)
+        n_round = len(inputs.messages) // 2
+        if n_round > 1 and not self.template_meta.support_multi_round:
+            logger.warning_once(
+                'The template does not support multi-round chat. Only use the last round of the conversation.')
+            inputs.messages = inputs.messages[-2:]
+
+        res_context_list: List[Context] = []
+        res_context_types: List[ContextType] = []
+        sep_token = None
+        if template_meta.auto_add_bos:
+            all_tokens = self.tokenizer.encode('a')
+            single_token = self.tokenizer.encode('a', add_special_tokens=False)
+            assert len(single_token) == 1
+            idx = all_tokens.index(single_token[0])
+            bos_token = all_tokens[:idx]
+            sep_token = all_tokens[idx + 1:]
+            if bos_token:
+                res_context_list.append(bos_token)
+                res_context_types.append(ContextType.OTHER)
+
+        prefix = template_meta.system_prefix if system else template_meta.prefix
+        self._concat_context_list(prefix, res_context_list, res_context_types, system=system)
+
+        n_round = len(inputs.messages) // 2
+        for i, (query_message, response_message) in enumerate(zip(inputs.messages[::2], inputs.messages[1::2])):
+            query_role, query = query_message['role'], query_message['content']
+            response_role, response = response_message['role'], response_message['content']
+            # TODO: Optimize the Template mechanism.
+            assert query_role in {'user', 'tool'}, f'query_role: {query_role}'
+            assert response_role in {'assistant'}, f'response_role: {response_role}'
+            if query_role == 'tool':
+                prompt = query
+                query = ''
+            elif template_meta.is_post_system and i == n_round - 1:
+                prompt = template_meta.system_prompt
+            else:
+                prompt = template_meta.prompt
+
+            context_list = prompt.copy()
+            extra_context_list = []
+            extra_context_type = None
+            if i < n_round - 1:
+                # Not the last round.
+                context_list.append('{{RESPONSE}}')
+                if inputs.messages[2 * (i + 1)]['role'] != 'tool':
+                    extra_context_list = template_meta.chat_sep
+                    extra_context_type = ContextType.OTHER
+            elif response is not None:
+                # It is the final round, and the response exists (during training).
+                context_list.append('{{RESPONSE}}')
+                if self.is_training and not sep_token:
+                    extra_context_list = template_meta.suffix
+                    extra_context_type = ContextType.SUFFIX
+            elif template_meta.response_prefix:
+                # final round and during inference.
+                context_list.append(template_meta.response_prefix)
+
+            self._concat_context_list(
+                context_list,
+                res_context_list,
+                res_context_types,
+                query=query,
+                response=response,
+                system=system,
+                round0=i)
+            res_context_list += extra_context_list
+            res_context_types += [extra_context_type] * len(extra_context_list)
+        if template_meta.auto_add_bos and sep_token:
+            res_context_list.append(sep_token)
+            res_context_types.append(ContextType.SUFFIX)
+        from swift.plugin import loss_scale_map
+        res_context_list, loss_scale_list = loss_scale_map[self.loss_scale](res_context_list, res_context_types,
+                                                                            inputs.messages)
+        if self.is_training:
+            answer_len = len(extra_context_list) + bool(response is not None)
+        else:
+            answer_len = 0
+        return res_context_list, loss_scale_list, answer_len
+
+    def _encode_truncated(self, inputs):
+        if self.mode in {'vllm', 'lmdeploy'}:
+            encoded = Template._encode(self, inputs)
+            for key in ['images', 'audios', 'videos']:
+                encoded[key] = getattr(inputs, key)
+        else:
+            encoded = self._encode(inputs)
+
+        input_ids = encoded.get('input_ids')
+        labels = encoded.get('labels')
+        loss_scale = encoded.get('loss_scale')
+        if self.max_length is not None:
+            if self.truncation_strategy == 'right':
+                input_ids = input_ids[:self.max_length]
+                if labels is not None:
+                    labels = labels[:self.max_length]
+                if loss_scale is not None:
+                    loss_scale = loss_scale[:self.max_length]
+            elif self.truncation_strategy == 'left':
+                if len(input_ids) > self.max_length:
+                    logger.warning_once(
+                        'Input data was left-truncated because its length exceeds `max_length` (input length: '
+                        f'{len(input_ids)}, max_length: {self.max_length}). '
+                        'This may cause loss of important tokens (e.g., image tokens) and lead to errors. '
+                        'To avoid this, consider increasing `max_length` or pre-filtering long sequences.',
+                        hash_id='max_length_check')
+                input_ids = input_ids[-self.max_length:]
+                if labels is not None:
+                    labels = labels[-self.max_length:]
+                if loss_scale is not None:
+                    loss_scale = loss_scale[-self.max_length:]
+            elif self.truncation_strategy == 'raise':
+                length = len(input_ids or labels or [])
+                if length > self.max_length:
+                    raise MaxLengthError(f'Current length of row({length}) is larger'
+                                         f' than the max_length({self.max_length}).')
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        return encoded
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        template_backend = self.template_backend
+        if (self.template_meta.template_type == 'dummy' and self.use_chat_template and not self.is_training
+                and self.mode != 'seq_cls'):
+            template_backend = 'jinja'
+            logger.info_once(f'Setting template_backend: {template_backend}')
+        res_context_list, loss_scale_list, answer_len = (
+            self._swift_encode(inputs) if template_backend == 'swift' else self._jinja_encode(inputs))
+        encoded = {}
+        if self.is_encoder_decoder:
+            # tokenizer_kwargs: use prompt (qwen-audio)
+            total_len = len(res_context_list)
+            for key, _slice in zip(['prompt', 'answer'],
+                                   [slice(0, total_len - answer_len),
+                                    slice(total_len - answer_len, total_len)]):
+                context_list, loss_scale = self._simplify_context_list(res_context_list[_slice],
+                                                                       loss_scale_list[_slice], inputs)
+                input_ids, labels, loss_scale, tokenizer_kwargs = self._encode_context_list(context_list, loss_scale)
+                encoded[f'{key}_input_ids'] = input_ids
+                if key == 'answer':
+                    encoded['labels'] = labels
+                    encoded['loss_scale'] = loss_scale
+            input_ids = encoded['prompt_input_ids'] + encoded['answer_input_ids']
+        else:
+            res_context_list, loss_scale_list = self._simplify_context_list(res_context_list, loss_scale_list, inputs)
+            input_ids, labels, loss_scale, tokenizer_kwargs = self._encode_context_list(
+                res_context_list, loss_scale_list)
+        self._add_dynamic_eos(input_ids, labels, loss_scale, self._encode_context_list(self.template_meta.suffix)[0])
+
+        if tokenizer_kwargs:
+            encoded['tokenizer_kwargs'] = tokenizer_kwargs
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['loss_scale'] = loss_scale
+        if self.use_megatron:
+            self._handle_megatron_cp(encoded)
+            encoded['labels'] = encoded['labels'][1:] + [-100]
+            encoded['position_ids'] = list(range(len(encoded['labels'])))
+        elif encoded.get('labels') is not None:
+            encoded['labels'][0] = -100
+        if not self.is_training:
+            for k in list(encoded.keys()):
+                if k.endswith('labels') or k.endswith('loss_scale'):
+                    encoded[k] = None
+        return encoded
+
+    def _handle_megatron_cp(self, encoded: Dict[str, Any]) -> None:
+        cp_size = self.sequence_parallel_size
+        if cp_size == 1:
+            return
+        input_ids = encoded['input_ids']
+        padding_len = math.ceil(len(input_ids) / (cp_size * 2)) * (cp_size * 2) - len(input_ids)
+        input_ids += [self.tokenizer.pad_token_id] * padding_len
+        encoded['labels'] += [-100] * padding_len
+
+    def debug_logger(self, inputs):
+        if not strtobool(os.getenv('SWIFT_DEBUG', 'false')):
+            return
+        if 'input_ids' in inputs:
+            k = 'input_ids'
+            val = inputs['input_ids']
+        elif 'generate_ids' in inputs:
+            k = 'generate_ids'
+            val = inputs['generate_ids']
+        for v in val:
+            self.print_inputs({k: v.tolist()})
+
+    @staticmethod
+    def _split_list(inputs: List[int], x: int) -> List[List[int]]:
+        idxs = findall(inputs, x)
+        idxs.append(len(inputs))
+        res = []
+        lo = 0
+        for idx in idxs:
+            res.append(inputs[lo:idx])
+            lo = idx + 1
+        return res
+
+    def replace_video2image(self, load_video_func, inputs, replace_tag: Callable) -> List[Context]:
+        context_list = []
+        if self.mode in {'vllm', 'lmdeploy'}:
+            video = inputs.videos.pop(inputs.video_idx)
+            inputs.video_idx -= 1
+        else:
+            video = inputs.videos[inputs.video_idx]
+        images = inputs.images
+        new_images = load_video_func(video)
+        inputs.images = images[:inputs.image_idx] + new_images + images[inputs.image_idx:]
+        for i in range(len(new_images)):
+            context_list += replace_tag(i)
+        inputs.image_idx += len(new_images)
+        return context_list
+
+    def get_generate_ids(self, generate_ids: Union[torch.Tensor, List[int]],
+                         num_prompt_tokens: int) -> Union[torch.Tensor, List[int]]:
+        if self.skip_prompt:
+            generate_ids = generate_ids[..., num_prompt_tokens:]
+        return generate_ids
+
+    def post_process_generate_response(self, response: str, inputs: StdTemplateInputs) -> str:
+        return response
+
+    def pre_forward_hook(self, model: nn.Module, args, kwargs):
+        from swift.llm import to_device
+        old_kwargs = to_device(kwargs, model.device)
+        kwargs = to_device(self._post_encode(model, old_kwargs), model.device)
+        for k, v in old_kwargs.items():
+            if k in {'input_ids', 'attention_mask', 'labels', 'position_ids'} and k not in kwargs:
+                kwargs[k] = v
+        if 'inputs_embeds' in kwargs:
+            kwargs.pop('input_ids', None)
+
+        if isinstance(model, PeftModel):
+            parameters = inspect.signature(model.model.forward).parameters
+        else:
+            parameters = inspect.signature(model.forward).parameters
+        if 'position_ids' not in parameters:
+            kwargs.pop('position_ids', None)
+        return args, kwargs
+
+    @property
+    def is_training(self):
+        return self.mode not in {'vllm', 'lmdeploy', 'pt'}
+
+    def set_mode(self, mode: Literal['vllm', 'lmdeploy', 'pt', 'seq_cls', 'train', 'rlhf', 'kto']) -> None:
+        self.mode = mode
+
+    def register_post_encode_hook(self, models: List[nn.Module]) -> None:
+        """This function is important for multi-modal training, as it registers the post_encode method
+            as a forward hook, converting input_ids into inputs_embeds.
+        """
+        if self._handles:
+            return
+
+        for model in models:
+            # please use torch>=2.0
+            handle = model.register_forward_pre_hook(self.pre_forward_hook, with_kwargs=True)
+            self._handles.append((model, handle))
+
+        if is_deepspeed_zero3_enabled():
+            import deepspeed
+            self._deepspeed_initialize = deepspeed.initialize
+
+            @wraps(self._deepspeed_initialize)
+            def _initialize(*args, **kwargs):
+                res = self._deepspeed_initialize(*args, **kwargs)
+                for model, handle in self._handles:
+                    model._forward_pre_hooks.move_to_end(handle.id)
+                return res
+
+            deepspeed.initialize = _initialize
+
+    def remove_post_encode_hook(self):
+        models = []
+        for model, handle in self._handles:
+            models.append(model)
+            handle.remove()
+        self._handles = []
+
+        if self._deepspeed_initialize is not None:
+            import deepspeed
+            deepspeed.initialize = self._deepspeed_initialize
+        self._deepspeed_initialize = None
+        return models
+
+    def data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        if self.mode == 'rlhf':
+            return self._rlhf_data_collator(batch, padding_to=padding_to)
+        elif self.mode == 'kto':
+            return self._kto_data_collator(batch, padding_to=padding_to)
+        elif self.mode in {'pt', 'train', 'prm'}:
+            return self._data_collator(batch, padding_to=padding_to)
+        elif self.mode == 'seq_cls':
+            return self._seq_cls_data_collator(batch, padding_to=padding_to)
+        elif self.mode == 'embedding':
+            return self._embedding_data_collator(batch, padding_to=padding_to)
+
+    @staticmethod
+    def _fetch_inputs_startswith(batch: List[Dict[str, Any]], prefix: str) -> List[Dict[str, Any]]:
+        new_batch = []
+        for inputs in batch:
+            new_inputs = {}
+            for k, v in inputs.items():
+                if k.startswith(prefix):
+                    new_inputs[k[len(prefix):]] = v
+            new_batch.append(new_inputs)
+        return new_batch
+
+    @staticmethod
+    def fetch_inputs(batch: List[Dict[str, Any]], keys: Optional[List[str]] = None) -> Dict[str, Any]:
+        from swift.llm import RowPreprocessor
+        keys = keys or []
+        rows = RowPreprocessor.rows_to_batched(batch)
+        return {k: rows[k] for k in keys if rows.get(k) is not None}
+
+    @staticmethod
+    def gather_list(batch: List[Dict[str, Any]], attr_name: str) -> Optional[List[Any]]:
+        # List[Tensor] ->  List[Tensor]
+        res = []
+        for b in batch:
+            if b.get(attr_name) is not None:
+                res += b.pop(attr_name)
+        return res
+
+    @staticmethod
+    def concat_tensor(batch: List[Dict[str, Any]], attr_name: str, dim: int) -> Optional[torch.Tensor]:
+        res = []
+        for b in batch:
+            if b.get(attr_name) is not None:
+                res.append(b.pop(attr_name))
+        return torch.concat(res, dim=dim) if res else None
+
+    def _rlhf_data_collator(self,
+                            batch: List[Dict[str, Any]],
+                            *,
+                            chosen_prefix: str = 'chosen_',
+                            rejected_prefix: str = 'rejected_',
+                            padding_to: Optional[int] = None) -> Dict[str, Any]:
+        new_batch = []
+        for prefix in [chosen_prefix, rejected_prefix]:
+            new_batch += self._fetch_inputs_startswith(batch, prefix)
+        return self._data_collator(new_batch, padding_to=padding_to)
+
+    def _kto_data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        new_batch = self._fetch_inputs_startswith(batch, 'chosen_')
+        kl_batch = self._fetch_inputs_startswith(batch, 'rejected_')
+
+        res = self._data_collator(new_batch, padding_to=padding_to)
+        kl_res = self._data_collator(kl_batch, padding_to=padding_to)
+        res = {
+            **{f'completion_{k}': v
+               for k, v in res.items()},
+            **{f'KL_completion_{k}': v
+               for k, v in kl_res.items()},
+        }
+        label = [b['label'] for b in batch if b.get('label') is not None]
+        if label:
+            res['label'] = label
+        return res
+
+    def _embedding_data_collator(self,
+                                 batch: List[Dict[str, Any]],
+                                 *,
+                                 padding_to: Optional[int] = None) -> Dict[str, Any]:
+        labels = []
+        new_batch = []
+        for b in batch:
+            keys = [key for key in b.keys() if 'negative' in key]
+            max_neg = max([int(re.findall(r'negative(-?\d+)', key)[0]) for key in keys]) if keys else None
+            indexes = ['anchor_', 'positive_']
+            if max_neg is not None:
+                for i in range(0, max_neg + 1):
+                    indexes.append(f'negative{i}_')
+            for prefix in indexes:
+                new_batch += self._fetch_inputs_startswith([b], prefix)
+            labels.extend(b.get('labels', None))
+        res = self._data_collator(new_batch, padding_to=padding_to)
+        if labels:
+            res['labels'] = torch.tensor(labels, dtype=torch.float32)
+        return res
+
+    def _seq_cls_data_collator(self,
+                               batch: List[Dict[str, Any]],
+                               *,
+                               padding_to: Optional[int] = None) -> Dict[str, Any]:
+        labels = [b.pop('labels') for b in batch if b.get('labels') is not None]
+        res = self._data_collator(batch, padding_to=padding_to)
+        if labels:
+            problem_type = self._get_problem_type(self.config)
+            if problem_type == 'regression':
+                labels = torch.tensor(labels, dtype=torch.float32)
+            elif problem_type == 'multi_label_classification':
+                one_hot_labels = torch.zeros((len(labels), self.config.num_labels), dtype=torch.float32)
+                for i, label in enumerate(labels):
+                    one_hot_labels[i, label] = 1
+                labels = one_hot_labels
+            else:
+                labels = torch.tensor(labels, dtype=torch.long)
+            res['labels'] = labels
+        return res
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        """
+        Args:
+            batch(`List[Dict[str, Any]]`): The input data in batch
+            padding_to(`int`, optional): Whether padding the batch to a fixed length, if none, the batch
+                will be padded to the `longest`
+        """
+        assert self.tokenizer.pad_token_id is not None
+        padding_side = self.padding_side if self.is_training else 'left'
+        padding_right = padding_side == 'right'
+        packing_mode = self.use_megatron or self._packing and 'position_ids' in batch[0]
+        res = {}
+        if packing_mode:
+            # only support llm
+            for k in ['input_ids', 'labels', 'position_ids', 'loss_scale']:
+                v = self.gather_list(batch, k)
+                if v:
+                    res[k] = [v]
+        else:
+            inputs_embeds = [b['inputs_embeds'] for b in batch if b.get('inputs_embeds') is not None]
+            input_ids = [b['input_ids'] for b in batch if b.get('input_ids') is not None]
+            if inputs_embeds:
+                res['inputs_embeds'] = inputs_embeds
+            if input_ids:
+                res['input_ids'] = input_ids
+            for key in ['labels', 'loss_scale', 'position_ids', 'token_type_ids']:
+                val = [b[key] for b in batch if b.get(key) is not None]
+                if val:
+                    res[key] = val
+
+        keys = [
+            'input_ids', 'inputs_embeds', 'attention_mask', 'labels', 'loss_scale', 'position_ids', 'token_type_ids'
+        ]
+        pad_values = [self.tokenizer.pad_token_id, 0., 0, -100, 0., 0., 0]
+        # Convert to tensor and remove unnecessary dimensions.
+        seq_lens = None
+        for key in keys:
+            if key not in res:
+                continue
+            for i, val in enumerate(res[key]):
+                if isinstance(val, (list, tuple)):
+                    val = torch.tensor(val)
+                elif key == 'inputs_embeds' and val.ndim == 3 or key != 'inputs_embeds' and val.ndim == 2:
+                    val = val[0]
+                res[key][i] = val
+            if not seq_lens:
+                seq_lens = [seq.shape[0] for seq in res[key]]
+        if not packing_mode and seq_lens and ('input_ids' in res or 'inputs_embeds' in res):
+            res['attention_mask'] = [torch.ones(seq_len, dtype=torch.int64) for seq_len in seq_lens]
+            if self.is_training and self.padding_side == 'left':
+                res['position_ids'] = [torch.arange(seq_len, dtype=torch.int64) for seq_len in seq_lens]
+
+        if self.use_megatron:
+            padding_to = math.ceil(max(seq_lens) / 128) * 128
+            cp_size = self.sequence_parallel_size
+            if cp_size > 1:
+                padding_len = padding_to - seq_lens[0]
+                position_ids = res['position_ids'][0].tolist()
+                position_ids += list(range(cp_size * 2)) * (padding_len // (cp_size * 2))
+                res['position_ids'][0] = torch.tensor(position_ids)
+
+        for key, pad_value in zip(keys, pad_values):
+            if key not in res:
+                continue
+            if self.use_megatron and key == 'position_ids' and self.sequence_parallel_size > 1:
+                pass
+            elif padding_to is not None:
+                padding_len = padding_to - seq_lens[0]
+                if padding_len > 0:
+                    res[key][0] = F.pad(res[key][0], (0, padding_len) if padding_right else (padding_len, 0),
+                                        'constant', pad_value)
+            res[key] = self._pad_sequence(res[key], pad_value)
+
+        # multimodal
+        res.update(self._data_collator_mm_data(batch))
+        if not self.use_megatron and (use_torchacc() or self.sequence_parallel_size > 1):
+            res = self._torchacc_xtuner_data_collator(res, padding_to, self.tokenizer, padding_side)
+
+        return res
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        # multimodal
+        res = {}
+        pixel_values = [b['pixel_values'] for b in batch if b.get('pixel_values') is not None]
+        if len(pixel_values) > 0:
+            res['pixel_values'] = torch.concat(pixel_values)
+
+            image_sizes = [b['image_sizes'] for b in batch if b.get('image_sizes') is not None]
+            if len(image_sizes) > 0:
+                res['image_sizes'] = torch.concat(image_sizes)
+
+        pixel_values_videos = [b['pixel_values_videos'] for b in batch if b.get('pixel_values_videos') is not None]
+        if len(pixel_values_videos) > 0:
+            res['pixel_values_videos'] = torch.concat(pixel_values_videos)
+        return res
+
+    def _torchacc_xtuner_data_collator(self, res, padding_to, tokenizer, padding_side):
+        # torchacc & xtuner
+        input_ids = res.get('input_ids')
+        attention_mask = res.get('attention_mask')
+        labels = res.get('labels')
+        loss_scale = res.get('loss_scale')
+        if use_torchacc():
+            from swift.utils.torchacc_utils import pad_and_split_batch
+            rank, _, world_size, _ = get_dist_setting()
+            input_ids, attention_mask, labels, loss_scale = pad_and_split_batch(
+                padding_to,
+                input_ids,
+                attention_mask,
+                labels,
+                loss_scale,
+                self.max_length,
+                tokenizer,
+                rank,
+                world_size,
+                padding_right=padding_side == 'right')
+        if self.sequence_parallel_size > 1 and input_ids is not None:
+            bs, seq_len = input_ids.shape
+            if 'position_ids' not in res:
+                position_ids = torch.arange(seq_len).unsqueeze(0).long().repeat(bs, 1)
+            else:
+                position_ids = res['position_ids']
+            assert padding_side == 'right' or bs == 1, 'Sequence parallel only support padding_side=right'
+            from swift.trainers.sequence_parallel import sequence_parallel
+            if sequence_parallel.world_size() > 1:
+                from swift.trainers.sequence_parallel import sequence_parallel
+                input_ids, _, labels, position_ids, attention_mask, loss_scale = \
+                    sequence_parallel.pad_and_split_inputs(
+                        tokenizer, input_ids, None, labels, position_ids, attention_mask, loss_scale)
+            res['position_ids'] = position_ids
+        _local_var = locals()
+        for key in ['input_ids', 'attention_mask', 'labels', 'loss_scale']:
+            value = _local_var[key]
+            if value is not None:
+                res[key] = value
+        return res
+
+    def print_inputs(self, inputs: Dict[str, Any], tokenizer_kwargs: Optional[Dict[str, Any]] = None) -> None:
+        if tokenizer_kwargs is None:
+            tokenizer_kwargs = {}
+        for key in [
+                'input', 'labels', 'generate', 'chosen_input', 'chosen_labels', 'rejected_input', 'rejected_labels'
+        ]:
+            val = inputs.get(key)  # fix val is a tensor
+            if val is None:
+                val = inputs.get(f'{key}_ids')
+            if val is not None:
+                key_upper = key.upper()
+                logger.info(f'[{key_upper}_IDS] {val}')
+                if key == 'labels' and self.mode in {'seq_cls', 'embedding'}:
+                    continue
+                if isinstance(val, (list, tuple, torch.Tensor)):
+                    val_str = self.safe_decode(val, **tokenizer_kwargs)
+                    logger.info(f'[{key_upper}] {val_str}')
+        if inputs.get('loss_scale') is not None:
+            val = inputs['loss_scale']
+            logger.info(f'[LOSS_SCALE] {val}')
+
+    async def prepare_lmdeploy_pytorch_inputs(self, inputs) -> None:
+        images = inputs.pop('images', None) or []
+        if len(images) == 0:
+            return
+        input_ids = inputs['input_ids']
+        idx_list = findall(input_ids, -100)
+        assert len(idx_list) == len(images), f'len(idx_list): {len(idx_list)}, len(images): {len(images)}'
+        idx_list.insert(0, -1)
+        new_input_ids = []
+        for i in range(len(idx_list) - 1):
+            new_input_ids += input_ids[idx_list[i] + 1:idx_list[i + 1]]
+            images[i]['offset'] = len(new_input_ids)
+            new_input_ids += [images[i]['image_token_id']] * images[i]['image_tokens']
+        new_input_ids += input_ids[idx_list[-1] + 1:]
+        inputs['input_ids'] = new_input_ids
+        inputs['multimodal'] = images
+
+    async def prepare_lmdeploy_turbomind_inputs(self, inputs: Dict[str, Any]) -> None:
+        images = inputs.pop('images', None) or []
+        if len(images) == 0:
+            return
+        from lmdeploy.vl.constants import IMAGE_DUMMY_TOKEN_INDEX
+        input_ids = inputs['input_ids']
+        idx_list = findall(input_ids, -100)
+        assert len(idx_list) == len(images), f'len(idx_list): {len(idx_list)}, len(images): {len(images)}'
+        idx_list.insert(0, -1)
+        new_input_ids = []
+        ranges = []
+        for i in range(len(idx_list) - 1):
+            _range = []
+            new_input_ids += input_ids[idx_list[i] + 1:idx_list[i + 1]]
+            _range.append(len(new_input_ids))
+            new_input_ids += [IMAGE_DUMMY_TOKEN_INDEX] * images[i].shape[0]
+            _range.append(len(new_input_ids))
+            ranges.append(_range)
+        new_input_ids += input_ids[idx_list[-1] + 1:]
+        inputs['input_embeddings'] = [image.to('cpu') for image in images]
+        inputs['input_embedding_ranges'] = ranges
+        inputs['input_ids'] = new_input_ids
+
+    def _pad_sequence(self, sequences: List[torch.Tensor], padding_value: float = 0.) -> torch.Tensor:
+        """Pad sequence by some side
+
+        Args:
+            sequences: The input sequences in tensor.
+            padding_value: The padding value
+
+        Returns:
+            A tensor after padding
+        """
+        padding_side = self.padding_side if self.is_training else 'left'
+        padding_right = padding_side == 'right'
+        if padding_right:
+            return pad_sequence(sequences, batch_first=True, padding_value=padding_value)
+
+        max_len = max([s.shape[0] for s in sequences])
+
+        padded_sequences = []
+        for seq in sequences:
+            pad_length = max_len - seq.shape[0]
+            pad_tuple = [0] * ((seq.dim() - 1) * 2) + [pad_length, 0]
+            padded_seq = F.pad(seq, tuple(pad_tuple), 'constant', padding_value)
+            padded_sequences.append(padded_seq)
+
+        return torch.stack(padded_sequences)
+
+    def safe_decode(self, input_ids: List[int], **tokenizer_kwargs) -> str:
+        if isinstance(self, Template):
+            tokenizer = self.tokenizer
+            placeholder_tokens = self.placeholder_tokens
+        else:
+            tokenizer = self
+            placeholder_tokens = []
+
+        def _is_special(token: int) -> bool:
+            if isinstance(token, float) or token < 0:
+                return True
+            return token in placeholder_tokens
+
+        if isinstance(input_ids, torch.Tensor):
+            input_ids = input_ids.tolist()
+        if len(input_ids) == 0:
+            return ''
+        result_str = ''
+        for i in range(len(input_ids)):
+            if i == 0:
+                if _is_special(input_ids[i]):
+                    s = 0
+                else:
+                    e = 0
+                continue
+            if _is_special(input_ids[i]) and not _is_special(input_ids[i - 1]):
+                s = i
+                result_str += tokenizer.decode(input_ids[e:s], **tokenizer_kwargs)
+            if not _is_special(input_ids[i]) and _is_special(input_ids[i - 1]):
+                e = i
+                result_str += f'[{input_ids[i - 1]} * {e - s}]'
+        if _is_special(input_ids[i]):
+            result_str += f'[{input_ids[i]} * {len(input_ids) - s}]'
+        else:
+            result_str += tokenizer.decode(input_ids[e:], **tokenizer_kwargs)
+        return result_str
+
+    @staticmethod
+    @contextmanager
+    def _patch_flash_attention_forward(modeling_module, position_ids, use_new_func: bool = False):
+        _origin_flash_attention_forward = modeling_module._flash_attention_forward
+
+        def _flash_attention_forward(*args, **kwargs):
+            if use_new_func:
+                from transformers.modeling_flash_attention_utils import (_flash_attention_forward as
+                                                                         flash_attention_forward)
+                if args and isinstance(args[0], nn.Module):
+                    args = args[1:]
+                if 'is_causal' not in kwargs:
+                    kwargs['is_causal'] = True
+            else:
+                flash_attention_forward = _origin_flash_attention_forward
+            kwargs['position_ids'] = position_ids
+            return flash_attention_forward(*args, **kwargs)
+
+        modeling_module._flash_attention_forward = _flash_attention_forward
+        try:
+            yield
+        finally:
+            modeling_module._flash_attention_forward = _origin_flash_attention_forward

swift/llm/template/constant.py

@@ -0,0 +1,194 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import List
+
+
+class LLMTemplateType:
+    chatml = 'chatml'
+    default = 'default'
+    dummy = 'dummy'
+
+    qwen = 'qwen'
+    qwen2_5 = 'qwen2_5'
+    qwen2_5_math = 'qwen2_5_math'
+    qwen2_5_math_prm = 'qwen2_5_math_prm'
+    qwen3 = 'qwen3'
+    qwq_preview = 'qwq_preview'
+    qwq = 'qwq'
+    marco_o1 = 'marco_o1'
+    modelscope_agent = 'modelscope_agent'
+
+    llama = 'llama'  # llama2
+    llama3 = 'llama3'
+    llama3_2 = 'llama3_2'
+    reflection = 'reflection'
+    megrez = 'megrez'
+    yi_coder = 'yi_coder'
+    sus = 'sus'
+
+    minimax = 'minimax'
+    minimax_vl = 'minimax_vl'
+
+    numina = 'numina'
+    ziya = 'ziya'
+    atom = 'atom'
+    mengzi = 'mengzi'
+
+    chatglm2 = 'chatglm2'
+    glm4 = 'glm4'
+    glm4_0414 = 'glm4_0414'
+    glm4_z1_rumination = 'glm4_z1_rumination'
+    codegeex4 = 'codegeex4'
+    longwriter_llama = 'longwriter_llama'
+
+    internlm = 'internlm'
+    internlm2 = 'internlm2'
+    internlm3 = 'internlm3'
+
+    deepseek = 'deepseek'
+    deepseek_coder = 'deepseek_coder'
+    deepseek_v2_5 = 'deepseek_v2_5'
+    deepseek_r1 = 'deepseek_r1'
+
+    openbuddy = 'openbuddy'
+    openbuddy2 = 'openbuddy2'
+    baichuan = 'baichuan'
+    baichuan_m1 = 'baichuan_m1'
+    minicpm = 'minicpm'
+    telechat = 'telechat'
+    telechat2 = 'telechat2'
+
+    codefuse = 'codefuse'
+    codefuse_codellama = 'codefuse_codellama'
+
+    skywork = 'skywork'
+    skywork_o1 = 'skywork_o1'
+
+    mistral_nemo = 'mistral_nemo'
+    mistral_2501 = 'mistral_2501'
+    zephyr = 'zephyr'
+    wizardlm2 = 'wizardlm2'
+    wizardlm2_moe = 'wizardlm2_moe'
+    gemma = 'gemma'
+    gemma3_text = 'gemma3_text'
+    phi3 = 'phi3'
+    phi4 = 'phi4'
+
+    ling = 'ling'
+    yuan = 'yuan'
+    xverse = 'xverse'
+    bluelm = 'bluelm'
+    orion = 'orion'
+    moonlight = 'moonlight'
+
+    aya = 'aya'
+    c4ai = 'c4ai'
+    dbrx = 'dbrx'
+
+
+class RMTemplateType:
+    internlm2_reward = 'internlm2_reward'
+
+
+class MLLMTemplateType:
+    qwen_vl = 'qwen_vl'
+    qwen_audio = 'qwen_audio'
+    qwen2_vl = 'qwen2_vl'
+    qwen2_5_vl = 'qwen2_5_vl'
+    qwen2_5_omni = 'qwen2_5_omni'
+    qwen2_audio = 'qwen2_audio'
+    qwen2_gme = 'qwen2_gme'
+    qvq = 'qvq'
+    ovis1_6 = 'ovis1_6'
+    ovis1_6_llama3 = 'ovis1_6_llama3'
+    ovis2 = 'ovis2'
+
+    llama3_1_omni = 'llama3_1_omni'
+    llama3_2_vision = 'llama3_2_vision'
+    llama4 = 'llama4'
+
+    llava1_5_hf = 'llava1_5_hf'
+    llava1_6_mistral_hf = 'llava1_6_mistral_hf'
+    llava1_6_vicuna_hf = 'llava1_6_vicuna_hf'
+    llava1_6_yi_hf = 'llava1_6_yi_hf'
+    llama3_llava_next_hf = 'llama3_llava_next_hf'
+    llava_next_qwen_hf = 'llava_next_qwen_hf'
+    llava_onevision_hf = 'llava_onevision_hf'
+    llava_next_video_hf = 'llava_next_video_hf'
+
+    llava_llama3_1_hf = 'llava_llama3_1_hf'  # DaozeZhang
+    llava_llama3_hf = 'llava_llama3_hf'  # xtuner
+    # lmms-lab
+    llava1_6_mistral = 'llava1_6_mistral'
+    llava1_6_yi = 'llava1_6_yi'
+    llava_next_qwen = 'llava_next_qwen'
+    llama3_llava_next = 'llama3_llava_next'
+
+    yi_vl = 'yi_vl'
+
+    internvl = 'internvl'
+    internvl_phi3 = 'internvl_phi3'
+    internvl2 = 'internvl2'
+    internvl2_phi3 = 'internvl2_phi3'
+    internvl2_5 = 'internvl2_5'
+
+    xcomposer2 = 'ixcomposer2'
+    xcomposer2_4khd = 'xcomposer2_4khd'
+    xcomposer2_5 = 'xcomposer2_5'
+
+    cogagent_chat = 'cogagent_chat'
+    cogagent_vqa = 'cogagent_vqa'
+    cogvlm = 'cogvlm'
+    cogvlm2 = 'cogvlm2'
+    cogvlm2_video = 'cogvlm2_video'
+    glm4v = 'glm4v'
+    glm_edge_v = 'glm_edge_v'
+
+    minicpmv = 'minicpmv'
+    minicpmv2_5 = 'minicpmv2_5'
+    minicpmv2_6 = 'minicpmv2_6'
+    minicpmo2_6 = 'minicpmo2_6'
+
+    deepseek_vl = 'deepseek_vl'
+    deepseek_vl2 = 'deepseek_vl2'
+    deepseek_janus = 'deepseek_janus'
+    deepseek_janus_pro = 'deepseek_janus_pro'
+
+    mplug_owl2 = 'mplug_owl2'
+    mplug_owl3 = 'mplug_owl3'
+    mplug_owl3_241101 = 'mplug_owl3_241101'
+    doc_owl2 = 'doc_owl2'
+
+    emu3_chat = 'emu3_chat'
+    emu3_gen = 'emu3_gen'
+
+    got_ocr2 = 'got_ocr2'
+    got_ocr2_hf = 'got_ocr2_hf'
+    step_audio = 'step_audio'
+    kimi_vl = 'kimi_vl'
+
+    idefics3 = 'idefics3'
+    pixtral = 'pixtral'
+    paligemma = 'paligemma'
+    phi3_vision = 'phi3_vision'
+    phi4_multimodal = 'phi4_multimodal'
+    florence = 'florence'
+    molmo = 'molmo'
+    megrez_omni = 'megrez_omni'
+    valley = 'valley'
+    gemma3_vision = 'gemma3_vision'
+    mistral_2503 = 'mistral_2503'
+
+
+class TemplateType(LLMTemplateType, MLLMTemplateType, RMTemplateType):
+
+    @classmethod
+    def get_template_name_list(cls) -> List[str]:
+        res = []
+        for k in cls.__dict__.keys():
+            if k.startswith('__'):
+                continue
+            value = cls.__dict__[k]
+            if isinstance(value, str):
+                res.append(value)
+        return res

swift/llm/template/grounding.py

@@ -0,0 +1,91 @@
+import colorsys
+import itertools
+import os
+from copy import deepcopy
+from typing import Any, List, Literal
+
+import requests
+from modelscope.hub.utils.utils import get_cache_dir
+from PIL import Image, ImageDraw, ImageFont
+
+
+def _shuffle_colors(nums: List[Any]) -> List[Any]:
+    if len(nums) == 1:
+        return nums
+
+    mid = len(nums) // 2
+
+    left = nums[:mid]
+    right = nums[mid:]
+    left = _shuffle_colors(left)
+    right = _shuffle_colors(right)
+    new_nums = []
+    for x, y in zip(left, right):
+        new_nums += [x, y]
+    new_nums += left[len(right):] or right[len(left):]
+    return new_nums
+
+
+def generate_colors():
+    vs_combinations = [(v, s) for v, s in itertools.product([0.7, 0.3, 1], [0.7, 0.3, 1])]
+    colors = [colorsys.hsv_to_rgb(i / 16, s, v) for v, s in vs_combinations for i in _shuffle_colors(list(range(16)))]
+    colors = [(int(r * 255), int(g * 255), int(b * 255)) for r, g, b in colors]
+    return _shuffle_colors(colors)
+
+
+def download_file(url: str) -> str:
+    url = url.rstrip('/')
+    file_name = url.rsplit('/', 1)[-1]
+    cache_dir = os.path.join(get_cache_dir(), 'files')
+    os.makedirs(cache_dir, exist_ok=True)
+    req = requests.get(url)
+    file_path = os.path.join(cache_dir, file_name)
+    with open(file_path, 'wb') as f:
+        f.write(req.content)
+    return file_path
+
+
+colors = generate_colors()
+color_mapping = {}
+
+
+def _calculate_brightness(image, region: List[int]):
+    cropped_image = image.crop(region)
+    grayscale_image = cropped_image.convert('L')
+    pixels = list(grayscale_image.getdata())
+    average_brightness = sum(pixels) / len(pixels)
+    return average_brightness
+
+
+def draw_bbox(image: Image.Image,
+              ref: List[str],
+              bbox: List[List[int]],
+              norm_bbox: Literal['norm1000', 'none'] = 'norm1000'):
+    bbox = deepcopy(bbox)
+    font_path = 'https://modelscope.cn/models/Qwen/Qwen-VL-Chat/resolve/master/SimSun.ttf'
+    # norm bbox
+    for i, box in enumerate(bbox):
+        for i in range(len(box)):
+            box[i] = int(box[i])
+        if norm_bbox == 'norm1000':
+            box[0] = box[0] / 1000 * image.width
+            box[2] = box[2] / 1000 * image.width
+            box[1] = box[1] / 1000 * image.height
+            box[3] = box[3] / 1000 * image.height
+
+    draw = ImageDraw.Draw(image)
+    # draw bbox
+    assert len(ref) == len(bbox), f'len(refs): {len(ref)}, len(bboxes): {len(bbox)}'
+    for (left, top, right, bottom), box_ref in zip(bbox, ref):
+        if box_ref not in color_mapping:
+            color_mapping[box_ref] = colors[len(color_mapping) % len(colors)]
+        color = color_mapping[box_ref]
+        draw.rectangle([(left, top), (right, bottom)], outline=color, width=3)
+    # draw text
+    file_path = download_file(font_path)
+    font = ImageFont.truetype(file_path, 20)
+    for (left, top, _, _), box_ref in zip(bbox, ref):
+        brightness = _calculate_brightness(
+            image, [left, top, min(left + 100, image.width),
+                    min(top + 20, image.height)])
+        draw.text((left, top), box_ref, fill='white' if brightness < 128 else 'black', font=font)

swift/llm/template/register.py

@@ -0,0 +1,57 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Dict, Literal, Optional
+
+from ..utils import Processor
+from .base import Template
+from .template_meta import TemplateMeta
+
+TEMPLATE_MAPPING: Dict[str, TemplateMeta] = {}
+
+
+def register_template(template_meta: TemplateMeta, *, exist_ok: bool = False) -> None:
+    template_type = template_meta.template_type
+    if not exist_ok and template_type in TEMPLATE_MAPPING:
+        raise ValueError(f'The `{template_type}` has already been registered in the TEMPLATE_MAPPING.')
+    TEMPLATE_MAPPING[template_type] = template_meta
+
+
+def get_template(
+        template_type: str,
+        processor: Processor,
+        default_system: Optional[str] = None,
+        max_length: Optional[int] = None,
+        *,
+        use_chat_template: bool = True,
+        template_backend: Literal['swift', 'jinja'] = 'swift',
+        truncation_strategy: Literal['raise', 'left', 'right'] = 'raise',
+        max_pixels: Optional[int] = None,  # h * w
+        agent_template: Optional[str] = None,
+        norm_bbox: Literal['norm1000', 'none', None] = None,
+        response_prefix: Optional[str] = None,
+        # train
+        padding_side: Literal['left', 'right'] = 'right',
+        loss_scale: str = 'default',
+        sequence_parallel_size: int = 1) -> 'Template':
+    template_meta = TEMPLATE_MAPPING[template_type]
+    template_cls = template_meta.template_cls
+    return template_cls(
+        processor,
+        template_meta,
+        default_system,
+        max_length,
+        use_chat_template=use_chat_template,
+        template_backend=template_backend,
+        truncation_strategy=truncation_strategy,
+        max_pixels=max_pixels,
+        agent_template=agent_template,
+        norm_bbox=norm_bbox,
+        response_prefix=response_prefix,
+        padding_side=padding_side,
+        loss_scale=loss_scale,
+        sequence_parallel_size=sequence_parallel_size,
+    )
+
+
+def get_template_meta(template_type: str) -> TemplateMeta:
+    return TEMPLATE_MAPPING[template_type]

swift/llm/template/template/__init__.py

@@ -0,0 +1,2 @@
+from . import (deepseek, emu3, gemma, glm, idefics3, internlm, internvl, llama, llava, llm, megrez, microsoft, minicpm,
+               minimax, mistral, molmo, moonshot, mplug, openbuddy, pixtral, qwen, stepfun, valley, yi)

swift/llm/template/template/deepseek.py

@@ -0,0 +1,315 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Optional
+
+import numpy as np
+import torch
+import torch.nn as nn
+from PIL import Image
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Prompt, findall
+
+
+@dataclass
+class DeepseekTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: [['bos_token_id']])
+    prompt: Prompt = field(default_factory=lambda: ['User: {{QUERY}}\n\nAssistant:'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: [['eos_token_id']])
+    suffix: Prompt = field(default_factory=lambda: [['eos_token_id']])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: [['bos_token_id'], '{{SYSTEM}}\n\n'])
+
+
+register_template(DeepseekTemplateMeta(LLMTemplateType.deepseek, ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.deepseek_coder,
+        prefix=['{{SYSTEM}}'],
+        prompt=['### Instruction:\n{{QUERY}}\n### Response:\n'],
+        chat_sep=['\n<|EOT|>\n'],
+        suffix=['\n<|EOT|>'],
+        stop_words=['<|EOT|>'],
+        default_system=('You are an AI programming assistant, utilizing the Deepseek Coder model, '
+                        'developed by Deepseek Company, and you only answer questions related to computer science. '
+                        'For politically sensitive questions, security and privacy issues, '
+                        'and other non-computer science questions, you will refuse to answer\n')))
+
+
+class DeepseekVLTemplate(Template):
+    image_placeholder = ['<image_placeholder>']
+    skip_prompt = False
+    use_model = True
+    placeholder_tokens = ['<image_placeholder>']
+
+    image_token_num_per_image: int = 576
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        is_janus = getattr(self, 'is_janus', False)
+
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        processor = self.processor
+        input_ids, labels = encoded['input_ids'], encoded['labels']
+
+        if not inputs.generate_mode:  # understanding task
+            idx_list = findall(input_ids, processor.image_id)  # '<image_placeholder>'
+            new_input_ids, new_labels = [], []
+            lo = 0
+            for hi in idx_list:
+                new_input_ids += input_ids[lo:hi]
+                if labels is not None:
+                    new_labels += labels[lo:hi]
+                image_tokens = [processor.image_id] * processor.num_image_tokens
+                if is_janus:
+                    image_tokens = [processor.image_start_id] + image_tokens + [processor.image_end_id]
+                new_input_ids += image_tokens
+                new_labels += [-100] * len(image_tokens)
+                lo = hi + 1
+            new_input_ids += input_ids[lo:]
+            if labels is not None:
+                new_labels += labels[lo:]
+            else:
+                new_labels = None
+            if is_janus:
+                from janus.models.processing_vlm import VLChatProcessorOutput
+            else:
+                from deepseek_vl.models.processing_vlm import VLChatProcessorOutput
+
+            images_outputs = processor.image_processor(images, return_tensors='pt')
+            output = VLChatProcessorOutput(
+                sft_format=None,
+                input_ids=torch.tensor(new_input_ids),
+                pixel_values=images_outputs.pixel_values,
+                num_image_tokens=torch.tensor([processor.num_image_tokens] * len(idx_list)))
+            encoded = {'output': output, 'input_ids': new_input_ids, 'labels': new_labels}
+            return encoded
+
+        else:  # image generation task
+            if self.is_training:
+                raise NotImplementedError('Only support the inference of generation of Janus series models.')
+            sft_format = self.tokenizer.decode(input_ids)
+            prompt = sft_format + processor.image_start_tag
+            input_ids = processor.tokenizer.encode(prompt)
+            input_ids = torch.LongTensor(input_ids)
+
+            encoded = {'input_ids': input_ids, 'labels': labels, 'generate_mode': inputs.generate_mode}
+            return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not inputs.get('generate_mode'):
+            inputs['pixel_values'] = inputs['pixel_values'].to(dtype=self.model_info.torch_dtype)
+            inputs_embeds = model.prepare_inputs_embeds(**inputs)
+            return {'inputs_embeds': inputs_embeds}
+        else:
+            return inputs
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        gene_img_list = [b.get('generate_mode') for b in batch]
+        if all(gene_img_list):
+            generate_mode = True
+        elif not any(gene_img_list):
+            generate_mode = False
+        else:
+            raise NotImplementedError('Do not support understanding and image generation tasks in one batch.')
+
+        if not generate_mode:
+            output = self.fetch_inputs(batch, ['output'])['output']
+            batched_output = dict(self.processor.batchify(output))
+            res = super()._data_collator(batch, padding_to=padding_to)
+            return {**batched_output, **res}
+        else:
+            res = super()._data_collator(batch, padding_to=padding_to)
+            res['generate_mode'] = generate_mode
+            return res
+
+    def generate(self, model, *args, **kwargs):
+        if not kwargs.get('generate_mode'):
+            return super().generate(model, *args, **kwargs)
+
+        else:
+            # generate how many number of images for each prompt, it is named parallel_size in the author's code
+            parallel_size = kwargs['generation_config'].num_return_sequences
+            temperature = kwargs['generation_config'].temperature
+            cfg_weight = get_env_args('cfg_weight', float, 5.0)
+
+            input_ids = kwargs['input_ids']  # [bsz, max_input_token_num]
+            bsz, max_input_token_num = input_ids.shape
+            tokens = torch.zeros((bsz, parallel_size * 2, max_input_token_num),
+                                 dtype=torch.int).cuda()  # [bsz, parallel_size*2, max_input_token_num]
+            for i in range(parallel_size * 2):
+                tokens[:, i, :] = input_ids
+                if i % 2 != 0:
+                    tokens[:, i, 1:-1] = self.processor.pad_id
+
+            inputs_embeds = model.language_model.get_input_embeddings()(
+                tokens)  # [bsz, parallel_size*2, max_input_token_num, 2048]
+
+            generated_tokens = torch.zeros(
+                (bsz, parallel_size, self.image_token_num_per_image),
+                dtype=torch.int).cuda()  # [bsz, 16, image_token_num_per_image] placeholder for the generated tokens
+
+            # set the first two dimensions into one dimension for batch size
+            inputs_embeds = inputs_embeds.reshape(bsz * parallel_size * 2, max_input_token_num, -1)
+            generated_tokens = generated_tokens.reshape(bsz * parallel_size, self.image_token_num_per_image)
+
+            for i in range(self.image_token_num_per_image):  # generate the tokens of image in a auto-regression way
+                outputs = model.language_model.model(
+                    inputs_embeds=inputs_embeds,
+                    use_cache=True,
+                    past_key_values=outputs.past_key_values if i != 0 else None)
+                hidden_states = outputs.last_hidden_state
+
+                logits = self.model.gen_head(hidden_states[:, -1, :])
+                logit_cond = logits[0::2, :]
+                logit_uncond = logits[1::2, :]
+
+                logits = logit_uncond + cfg_weight * (logit_cond - logit_uncond)
+                probs = torch.softmax(logits / temperature, dim=-1)
+
+                next_token = torch.multinomial(probs, num_samples=1)
+                generated_tokens[:, i] = next_token.squeeze(dim=-1)  # [parallel_size, self.image_token_num_per_image]
+
+                next_token = torch.cat([next_token.unsqueeze(dim=1), next_token.unsqueeze(dim=1)], dim=1).view(-1)
+                img_embeds = model.prepare_gen_img_embeds(next_token)  # [parallel_size * 2, 2048]
+                inputs_embeds = img_embeds.unsqueeze(dim=1)  # [parallel_size * 2, 1, 2048]
+
+            # no need to reset the original first two dimensions, waiting for the update of the upper layer
+            # inputs_embeds = inputs_embeds.reshape(bsz, parallel_size*2, -1)
+            # generated_tokens = generated_tokens.reshape(bsz, parallel_size, self.image_token_num_per_image)
+
+            return {'sequences': generated_tokens}
+
+    def decode(self, generate_ids: List[int], **kwargs) -> Any:
+        if 'template_inputs' not in kwargs or not kwargs['template_inputs'].generate_mode:
+            return super().decode(generate_ids, **kwargs)
+        else:
+            img_size = get_env_args('img_size', int, 384)
+            patch_size = 16
+
+            num_to_decode = 1  # for now, generate_ids is a 1D list
+
+            generate_ids = torch.tensor(generate_ids).unsqueeze(0)  # [num_to_decode=1, self.image_token_num_per_image]
+
+            dec = self.model.gen_vision_model.decode_code(
+                generate_ids.to(dtype=torch.int),
+                shape=[num_to_decode, 8, img_size // patch_size, img_size // patch_size])
+            dec = dec.to(torch.float32).cpu().numpy().transpose(0, 2, 3, 1)  # [num_to_decode, H, W, ch=3]
+
+            dec = np.clip((dec + 1) / 2 * 255, 0, 255)
+
+            visual_img = np.zeros((num_to_decode, img_size, img_size, 3), dtype=np.uint8)
+            visual_img[:, :, :] = dec
+
+            img_list = []
+            for i in range(num_to_decode):
+                cur_img = Image.fromarray(visual_img[i])
+                img_list.append({'type': 'image', 'image': cur_img})
+            return img_list
+
+
+@dataclass
+class DeepseekVLTemplateMeta(DeepseekTemplateMeta):
+    default_system: Optional[str] = ('You are a helpful language and vision assistant. '
+                                     'You are able to understand the visual content that the user provides, '
+                                     'and assist the user with a variety of tasks using natural language.')
+
+
+register_template(DeepseekVLTemplateMeta(
+    MLLMTemplateType.deepseek_vl,
+    template_cls=DeepseekVLTemplate,
+))
+
+
+class DeepseekJanus(DeepseekVLTemplate):
+    is_janus = True
+    image_placeholder = ['<image_placeholder>\n']
+
+
+register_template(DeepseekVLTemplateMeta(MLLMTemplateType.deepseek_janus, template_cls=DeepseekJanus))
+
+
+@dataclass
+class DeepseekV2_5TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<｜begin▁of▁sentence｜>{{SYSTEM}}'])
+    prompt: Prompt = field(default_factory=lambda: ['<｜User｜>{{QUERY}}<｜Assistant｜>'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<｜end▁of▁sentence｜>'])
+    suffix: Prompt = field(default_factory=lambda: ['<｜end▁of▁sentence｜>'])
+
+
+register_template(DeepseekV2_5TemplateMeta(LLMTemplateType.deepseek_v2_5))
+
+
+class DeepseekR1Template(Template):
+
+    def _swift_encode(self, inputs: StdTemplateInputs):
+        if not self.is_training:
+            for message in inputs.messages:
+                if message['role'] == 'assistant' and isinstance(message['content'], str):
+                    message['content'] = message['content'].split('</think>')[-1]
+        return super()._swift_encode(inputs)
+
+
+register_template(
+    DeepseekV2_5TemplateMeta(LLMTemplateType.deepseek_r1, template_cls=DeepseekR1Template, response_prefix='<think>\n'))
+
+
+class DeepseekVL2Template(DeepseekVLTemplate):
+    image_placeholder = ['<image>\n']
+    placeholder_tokens = ['<image>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from deepseek_vl2.models.processing_deepseek_vl_v2 import VLChatProcessorOutput
+        encoded = Template._encode(self, inputs)
+        images = inputs.images
+        processor = self.processor
+        input_ids, labels = encoded['input_ids'], encoded['labels']
+        images_seq_mask = [False] * len(input_ids)
+        idx_list = findall(input_ids, processor.image_token_id)  # '<image>'
+        _, images_list, _, images_spatial_crop, num_image_tokens = processor.tokenize_with_images(
+            '<image>' * len(images), images, cropping=len(images) <= 2)
+        new_num_tokens = 0
+        for idx, n_image_tokens in zip(idx_list, num_image_tokens):
+            image_tokens = [processor.image_token_id] * n_image_tokens
+            input_ids = input_ids[:idx] + image_tokens + input_ids[idx + 1:]
+            if labels is not None:
+                labels = labels[:idx] + [-100] * n_image_tokens + labels[idx + 1:]
+            images_seq_mask = images_seq_mask[:idx] + [True] * n_image_tokens + images_seq_mask[idx + 1:]
+            new_num_tokens += n_image_tokens - 1
+
+        output = VLChatProcessorOutput(
+            sft_format=None,
+            input_ids=torch.tensor(input_ids),
+            target_ids=torch.tensor(input_ids),
+            images=torch.stack(images_list) if images_list else torch.zeros((0, 3, 384, 384)),
+            images_seq_mask=torch.tensor(images_seq_mask),
+            images_spatial_crop=torch.tensor(images_spatial_crop),
+            num_image_tokens=num_image_tokens)
+        output.images = output.images.to(dtype=self.model_info.torch_dtype)
+        encoded = {'output': output, 'input_ids': input_ids, 'labels': labels}
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        inputs['images_seq_mask'] = inputs['images_seq_mask'].to(torch.bool)
+        inputs['images_spatial_crop'] = inputs['images_spatial_crop'].to(torch.long)
+        inputs_embeds = model.prepare_inputs_embeds(**inputs)
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(
+    DeepseekV2_5TemplateMeta(
+        MLLMTemplateType.deepseek_vl2,
+        prompt=['<|User|>: {{QUERY}}\n\n<|Assistant|>:'],
+        template_cls=DeepseekVL2Template,
+    ))
+
+register_template(
+    DeepseekVLTemplateMeta(
+        MLLMTemplateType.deepseek_janus_pro,
+        prompt=['<|User|>: {{QUERY}}\n\n<|Assistant|>:'],
+        template_cls=DeepseekJanus))

swift/llm/template/template/emu3.py

@@ -0,0 +1,191 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import random
+from typing import Any, Dict, List, Optional
+
+import torch
+from PIL import Image
+
+from swift.utils import get_device
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import register_template
+from ..template_inputs import StdTemplateInputs
+from ..template_meta import TemplateMeta
+from ..utils import findall
+from .utils import DEFAULT_SYSTEM, EmptyTemplateMeta
+
+
+class Emu3GenTemplate(Template):
+
+    NULL_PROMPT_PROB = 0.1
+    COOKBOOK_SIZE = 32768
+    CFG_SCALE = os.environ.get('CFG_SCALE', 3.0)
+    GENERATION_RATIO = os.environ.get('GENERATION_RATIO', '1:1')
+    NEGATIVE_PROMPT = os.environ.get(
+        'NEGATIVE_PROMPT',
+        'lowres, bad anatomy, bad hands, text, error, missing fingers, extra digit, fewer digits, cropped, '
+        'worst quality, low quality, normal quality, jpeg artifacts, signature, watermark, username, blurry.')
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.bov = self.processor.tokenizer.encode(self.processor.visual_template[0].format(token_id=0))[0]
+        self.eov = self.processor.tokenizer.encode(self.processor.visual_template[0].format(token_id=self.COOKBOOK_SIZE
+                                                                                            - 1))[0]
+        self.h, self.w = self.processor.calculate_generate_size(self.GENERATION_RATIO, self.processor.image_area,
+                                                                self.processor.vision_tokenizer.spatial_scale_factor)
+        self.skip_prompt = False
+        self.apply_loss_on_only_vision = True
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        if self.is_training:
+            p_prob = random.random()
+            if p_prob < self.NULL_PROMPT_PROB:
+                prompt = ''
+            else:
+                prompt = inputs.to_history()['response']
+            image = self.smart_resize(inputs.images[0].convert('RGB'))
+            with torch.no_grad():
+                image = self.processor.image_processor(
+                    image, return_tensors='pt')['pixel_values'].to(device=self.processor.vision_tokenizer.device)
+                image_token_ids = self.processor.vision_tokenizer.encode(image).squeeze(0)
+            encoded = self._process_prompt_train(prompt, image_token_ids)
+        else:
+            prompt = inputs.to_history()['query']
+            encoded = self._process_prompt_test(prompt)
+            encoded = {key: encoded[key][0] for key in encoded.keys()}  # [1, L] -> [L]
+
+        return encoded
+
+    def _process_prompt_train(self, raw_prompt, image_token_ids):
+        image_prompt = self.format_image_prompt(image_token_ids)
+        prompt = self.tokenizer.bos_token + raw_prompt + image_prompt
+        sample = self.tokenizer(prompt, padding='max_length', return_token_type_ids=False)
+        labels = torch.tensor(sample['input_ids'])
+        if self.apply_loss_on_only_vision:
+            labels = torch.where(torch.logical_and(labels >= self.bov, labels <= self.eov), labels, -100)
+        sample['labels'] = labels.tolist()
+        return sample
+
+    def _process_prompt_test(self, raw_prompt):
+        # for supporting multi inputs, use list instead of single string
+        if isinstance(raw_prompt, str):
+            raw_prompt = [raw_prompt]
+        prompt_list = []
+        size_list = []
+        for text_prompt in raw_prompt:
+            prompt = self.processor.tokenizer.bos_token
+            image_prompt = (
+                self.processor.tokenizer.boi_token + self.processor.prefix_template.format(H=self.h, W=self.w)
+                + self.processor.tokenizer.img_token)
+            prompt += (text_prompt + image_prompt)
+            prompt_list.append(prompt)
+            size_list.append([self.h, self.w])
+        prompt_list = self.tokenizer(prompt_list, padding='longest', return_token_type_ids=False)
+        return prompt_list
+
+    def prepare_for_output(self, output: str) -> str:
+        return output
+
+    def prepare_generate_kwargs(self, generate_kwargs: Dict[str, Any], *, model=None) -> Dict[str, Any]:
+        from transformers import UnbatchedClassifierFreeGuidanceLogitsProcessor
+        from transformers import PrefixConstrainedLogitsProcessor
+        from transformers import LogitsProcessorList
+
+        negative_prompt = self.NEGATIVE_PROMPT
+        neg_inputs = self._process_prompt_test(negative_prompt)
+        neg_inputs = {key: torch.tensor(val) for key, val in neg_inputs.items()}
+        batch_size = generate_kwargs['input_ids'].shape[0]
+        h = torch.tensor([self.h] * batch_size)
+        w = torch.tensor([self.w] * batch_size)
+
+        constrained_fn = self.processor.build_prefix_constrained_fn(h, w)
+        logits_processor = LogitsProcessorList([
+            UnbatchedClassifierFreeGuidanceLogitsProcessor(
+                self.CFG_SCALE,
+                model,
+                unconditional_ids=neg_inputs['input_ids'].to(get_device()),
+            ),
+            PrefixConstrainedLogitsProcessor(
+                constrained_fn,
+                num_beams=1,
+            ),
+        ])
+        res = super().prepare_generate_kwargs(generate_kwargs, model=model)
+        res['logits_processor'] = logits_processor
+        return res
+
+    def decode(self, generate_ids: List[int], **kwargs) -> Any:
+        mm_list = self.processor.decode(generate_ids)
+        for im in mm_list:
+            if not isinstance(im, Image.Image):
+                continue
+            return [{'type': 'image', 'image': im}]
+
+    def to_imgstr(self, image_tokens):
+        image_token_str = [[self.processor.visual_template[0].format(token_id=token_id) for token_id in token_row]
+                           for token_row in image_tokens]
+        image_row_str = [''.join(token_row) for token_row in image_token_str]
+        imgstr = self.tokenizer.eol_token.join(image_row_str)
+        return imgstr
+
+    def format_image_prompt(self, image_tokens):
+        h, w = image_tokens.shape
+        imgstr = self.to_imgstr(image_tokens)
+        image_prompt = (
+            self.tokenizer.boi_token + f'{h}*{w}' + self.tokenizer.img_token + imgstr + self.tokenizer.eol_token
+            + self.tokenizer.eof_token + self.tokenizer.eoi_token)
+        return image_prompt
+
+    def smart_resize(self, image):
+        w, h = image.size
+        current_area = h * w
+        target_ratio = (self.processor.image_area / current_area)**0.5
+        th = int(round(h * target_ratio))
+        tw = int(round(w * target_ratio))
+        image = image.resize((tw, th))
+        return image
+
+
+register_template(EmptyTemplateMeta(
+    MLLMTemplateType.emu3_gen,
+    template_cls=Emu3GenTemplate,
+))
+
+
+class Emu3ChatTemplate(Template):
+    system = 'You are a helpful assistant.'
+    image_placeholder = ['<|image token|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        # image
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        image_tokens = self.processor.tokenize_image(images)
+        image_prompts = []
+        idx_list = findall(input_ids, self.tokenizer.encode(self.image_placeholder))
+        # Create image prompts
+        for i in range(len(images)):
+            h, w = image_tokens[i].shape
+            imgstr = self.processor.to_imgstr(image_tokens[i])
+            image_prompt = (
+                self.tokenizer.boi_token + self.processor.prefix_template.format(H=h, W=w) + self.tokenizer.img_token
+                + imgstr + self.tokenizer.eol_token + self.tokenizer.eof_token + self.tokenizer.eoi_token)
+            image_prompts.append(self.tokenizer.encode(image_prompt))
+
+        # Insert image tokens into input_ids
+        input_ids, labels = self._extend_tokens(input_ids, labels, idx_list, lambda i: image_prompts[i])
+        return {'input_ids': input_ids, 'labels': labels}
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.emu3_chat,
+        prefix=[['bos_token_id'], '{{SYSTEM}}'],
+        prompt=[' User: {{QUERY}}. Assistant:'],
+        chat_sep=[['eos_token_id']],
+        suffix=[['eos_token_id']],
+        default_system=DEFAULT_SYSTEM,
+        template_cls=Emu3ChatTemplate))

swift/llm/template/template/gemma.py

@@ -0,0 +1,131 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+import numpy as np
+import torch
+
+from swift.utils import upper_bound
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+
+
+@dataclass
+class GemmaTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<bos>'])
+    prompt: Prompt = field(
+        default_factory=lambda: ['<start_of_turn>user\n{{QUERY}}<end_of_turn>\n<start_of_turn>model\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<end_of_turn>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<end_of_turn>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<bos><start_of_turn>system\n{{SYSTEM}}<end_of_turn>\n'])
+
+
+register_template(GemmaTemplateMeta(LLMTemplateType.gemma))
+
+
+class PaliGemmaTemplate(Template):
+    placeholder_tokens = ['<image>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        if self.mode == 'vllm':
+            self.prompt = ['{{QUERY}}']
+            return []
+        else:
+            self.prompt = ['{{QUERY}}\n']
+            return ['<image>' * self.processor.image_seq_length + '<bos>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        raw_image = inputs.images
+        processor = self.processor
+        if encoded['labels'] is not None:
+            n = upper_bound(0, len(encoded['labels']), lambda idx: encoded['labels'][idx] == -100)
+            n2 = len(encoded['labels']) - n
+            encoded['token_type_ids'] = [0] * n + [1] * n2
+        else:
+            encoded['token_type_ids'] = [0] * len(encoded['input_ids'])
+        if raw_image:
+            model_inputs = processor(text='<image>' * len(raw_image), images=raw_image, return_tensors='pt')
+            encoded['pixel_values'] = model_inputs['pixel_values'].to(self.model_info.torch_dtype)
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.paligemma,
+        prefix=[],
+        prompt=['{{QUERY}}\n'],
+        chat_sep=None,
+        suffix=['<eos>'],
+        template_cls=PaliGemmaTemplate,
+    ))
+
+
+@dataclass
+class Gemma3TextTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<bos>'])
+    prompt: Prompt = field(
+        default_factory=lambda: ['<start_of_turn>user\n{{QUERY}}<end_of_turn>\n<start_of_turn>model\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<end_of_turn>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<end_of_turn>'])
+
+
+class Gemma3Template(Template):
+
+    def _swift_encode(self, inputs: StdTemplateInputs):
+        if inputs.system is not None:
+            system = inputs.system
+            inputs.system = None
+            inputs.messages[0]['content'] = system + '\n\n' + inputs.messages[0]['content']
+        for message in inputs.messages:
+            if message['role'] == 'assistant' and isinstance(message['content'], str):
+                message['content'] = message['content'].strip('\n')
+        return super()._swift_encode(inputs)
+
+
+register_template(Gemma3TextTemplateMeta(LLMTemplateType.gemma3_text, template_cls=Gemma3Template))
+
+
+class Gemma3VisionTemplate(Gemma3Template):
+    boi_token_id = 255999
+    placeholder_tokens = ['<start_of_image>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<start_of_image>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from transformers.models.gemma3.processing_gemma3 import Gemma3ProcessorKwargs
+
+        encoded = super()._encode(inputs)
+        if inputs.images:
+            input_ids = encoded['input_ids']
+            labels = encoded['labels']
+            idx_list = findall(input_ids, self.boi_token_id)
+            img_tokens = self._tokenize(self.processor.full_image_sequence)
+            input_ids, labels = self._extend_tokens(input_ids, labels, idx_list, lambda _: img_tokens)
+
+            # TODO: customize
+            processor_kwargs = Gemma3ProcessorKwargs._defaults['images_kwargs']
+            image_inputs = self.processor.image_processor(inputs.images, **processor_kwargs)
+            image_inputs['pixel_values'] = torch.as_tensor(np.array(image_inputs['pixel_values']))
+            image_inputs.pop('num_crops')
+
+            array_ids = np.array(input_ids)
+            mm_token_type_ids = np.zeros_like(input_ids)
+            mm_token_type_ids[array_ids == self.processor.image_token_id] = 1
+            encoded['token_type_ids'] = mm_token_type_ids.tolist()
+            encoded['input_ids'] = input_ids
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            encoded['labels'] = labels
+        return encoded
+
+
+register_template(GemmaTemplateMeta(MLLMTemplateType.gemma3_vision, template_cls=Gemma3VisionTemplate))

swift/llm/template/template/glm.py

@@ -0,0 +1,293 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+import torch
+
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, Word, findall
+from ..vision_utils import load_batch, load_video_cogvlm2
+
+
+@dataclass
+class GLMTemplateMeta(TemplateMeta):
+    auto_add_bos: bool = True
+
+
+class GLM4Template(Template):
+
+    def _swift_encode(self, inputs: StdTemplateInputs):
+        res_context_list, loss_scale_list, answer_len = super()._swift_encode(inputs)
+        for i, res_context in enumerate(res_context_list):
+            # The last round or is tool_call.
+            if isinstance(res_context, str) and res_context.endswith('<|assistant|>\n') and (
+                    i + 1 >= len(res_context_list) or '<|observation|>' in res_context_list[i + 1]):
+                res_context_list[i] = res_context_list[i][:-len('\n')]
+        return res_context_list, loss_scale_list, answer_len
+
+    def decode(self, *args, **kwargs):
+        response = super().decode(*args, **kwargs)
+        return response.lstrip('\n')
+
+
+class GLM4_0414Template(GLM4Template):
+
+    def _swift_encode(self, inputs: StdTemplateInputs):
+        if not self.is_training:
+            for message in inputs.messages:
+                if message['role'] == 'assistant' and isinstance(message['content'], str):
+                    message['content'] = message['content'].split('</think>')[-1].strip()
+        return super()._swift_encode(inputs)
+
+
+register_template(
+    GLMTemplateMeta(
+        LLMTemplateType.chatglm2,
+        prefix=['{{SYSTEM}}'],
+        prompt=['[Round {{ROUND1}}]\n\n问：{{QUERY}}\n\n答：'],
+        chat_sep=['\n\n']))
+
+
+@dataclass
+class GLM4TemplateMeta(GLMTemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: ['<|user|>\n{{QUERY}}<|assistant|>\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=list)
+    suffix: Prompt = field(default_factory=lambda: ['<|user|>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<|system|>\n{{SYSTEM}}'])
+
+    agent_template: str = 'glm4'
+    stop_words: List[Word] = field(default_factory=lambda: ['<|endoftext|>', '<|user|>', '<|observation|>'])
+
+
+@dataclass
+class GLM4_0414TemplateMeta(GLM4TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['[gMASK]<sop>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['[gMASK]<sop><|system|>\n{{SYSTEM}}'])
+    agent_template: str = 'glm4_0414'
+
+
+class GLM4VTemplate(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return [[-100]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, -100)
+        if idx_list:
+            idx = idx_list[0]
+            image = inputs.images[0]
+            placeholder = '<|begin_of_image|><|endoftext|><|end_of_image|>'
+            placeholder_id = self.processor.encode(placeholder, add_special_tokens=False)
+            input_ids = (input_ids[:idx] + placeholder_id + input_ids[idx + 1:])
+            if labels is not None:
+                labels = (labels[:idx] + [-100] * len(placeholder_id) + labels[idx + 1:])
+            messages = inputs.messages
+            messages[0]['image'] = image
+            inputs2: Dict[str, Any] = self.processor.apply_chat_template(messages, return_dict=True)
+            encoded['images'] = inputs2['images']
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded['position_ids'] = list(range(len(input_ids)))
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = [b['images'] for b in batch if 'images' in b]
+        if images:
+            res['images'] = torch.concat(images)
+        return res
+
+
+register_template(GLM4TemplateMeta(MLLMTemplateType.glm4v, template_cls=GLM4VTemplate, suffix=['<|endoftext|>']))
+
+register_template(GLM4TemplateMeta(LLMTemplateType.glm4, template_cls=GLM4Template))
+
+register_template(GLM4_0414TemplateMeta(LLMTemplateType.glm4_0414, template_cls=GLM4_0414Template))
+
+glm4z1rumination_system = (
+    '你是一个专业的深度研究助手，通过提供的工具与模拟浏览器交互，来帮助用户完成深度信息调研和报告撰写任务。'
+    '今年是 2025 年。\n\n'
+    '<核心要求>\n'
+    '- 首先分解用户请求，得到包含多个子要求的列表\n'
+    '- 制定初始研究计划\n'
+    '- 进行多轮迭代搜索和页面浏览（at least 10 function calls）：\n'
+    '    * 根据已获得的信息调整研究计划和关键词\n'
+    '    * 打开页面阅读，从发现的内容中识别新的关键概念/名词\n'
+    '    * 从搜索结果中提取新的关键词继续搜索\n'
+    '    * 访问并仔细阅读相关页面，识别新的关键概念/名词\n\n'
+    '<重要配置>\n'
+    '- 采用语言\n'
+    '    * 搜索关键词：英语\n'
+    '    * 思考：英语\n\n'
+    '<可调用的工具列表>\n\n'
+    '[{"name": "search", "description": "Execute a search query and return search results. '
+    'Use this function when you need to find information about a specific topic.", '
+    '"parameters": {"type": "object", "properties": {"query": {"type": "string", '
+    '"description": "Search query string, use English words unless it is a proper name in Chinese"}}, '
+    '"required": ["query"], "additionalProperties": false}}, '
+    '{"name": "click", "description": "Click a link in the search results and navigate to the corresponding page. '
+    'Use this function when you need to view detailed content of a specific search result.", '
+    '"parameters": {"type": "object", "properties": {"link_id": {"type": "integer", '
+    '"description": "The link ID to click (from the sequence number in search results)"}}, '
+    '"required": ["link_id"], "additionalProperties": false}}, '
+    '{"name": "open", "description": "Open a specific website. Get content from any website with its URL.", '
+    '"parameters": {"type": "object", "properties": {"url": {"type": "string", '
+    '"description": "The target website URL or domain"}}, "required": ["url"], "additionalProperties": false}}, '
+    '{"name": "finish", "description": "Finish the task. '
+    'Use this function when you have found the information you need.", '
+    '"parameters": {"type": "object", "properties": {}, "additionalProperties": false}}]')
+
+register_template(
+    GLM4_0414TemplateMeta(
+        LLMTemplateType.glm4_z1_rumination, template_cls=GLM4_0414Template, default_system=glm4z1rumination_system))
+
+codegeex4_system = '你是一位智能编程助手，你叫CodeGeeX。你会为用户回答关于编程、代码、计算机方面的任何问题，并提供格式规范、可以执行、准确安全的代码，并在必要时提供详细的解释。'
+
+register_template(GLM4TemplateMeta(LLMTemplateType.codegeex4, default_system=codegeex4_system))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.longwriter_llama, ['[INST]'], ['{{QUERY}}[/INST]'], ['[INST]'], ['<|end_of_text|>'],
+        system_prefix=['<<SYS>>\n{{SYSTEM}}\n<</SYS>>\n\n']))
+
+
+class CogTemplate(Template):
+    placeholder_tokens = ['<|reserved_special_token_0|>']
+
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        return []
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        model = self.model
+        image = inputs.images or []
+        history_inputs = inputs.to_history()
+        inputs2 = model.build_conversation_input_ids(
+            self.processor, query=history_inputs['query'], history=history_inputs['history'], images=image)
+        image_token_len = inputs2['token_type_ids'].sum().item()
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        encoded['token_type_ids'] = [0] + [1] * image_token_len + [0] * len(input_ids[1:])
+        encoded['input_ids'] = input_ids[:1] + [self.processor.pad_token_id] * image_token_len + input_ids[1:]
+        if labels is not None:
+            encoded['labels'] = labels[:1] + [-100] * image_token_len + labels[1:]
+        if len(image) > 0:
+            encoded['images'] = [[img.to(dtype=self.model_info.torch_dtype)] for img in inputs2['images']]
+            if 'cross_images' in inputs2:
+                # is cogagent
+                encoded['cross_images'] = [[cross_img.to(dtype=self.model_info.torch_dtype)]
+                                           for cross_img in inputs2['cross_images']]
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        keys = ['images', 'cross_images']
+        for key in keys:
+            if key in batch[0]:
+                res[key] = [b[key][0] for b in batch]
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.cogagent_chat,
+        prefix=['<s>'],
+        prompt=[' [INST] {{QUERY}} [/INST] '],
+        chat_sep=[],
+        suffix=['</s>'],
+        template_cls=CogTemplate,
+    ))
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.cogagent_vqa,
+        prefix=['<s>'],
+        prompt=['<EOI>Question: {{QUERY}} Answer:'],
+        chat_sep=None,
+        suffix=['</s>'],
+        template_cls=CogTemplate))
+
+
+@dataclass
+class CogVLMTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: [['bos_token_id']])
+    prompt: Prompt = field(default_factory=lambda: ['Question: {{QUERY}} Answer:'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['\n'])
+
+
+register_template(CogVLMTemplateMeta(MLLMTemplateType.cogvlm, template_cls=CogTemplate))
+
+register_template(CogVLMTemplateMeta(MLLMTemplateType.cogvlm2, template_cls=CogTemplate))
+
+
+class Cog2VideoTemplate(CogTemplate):
+    use_model = True
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        model = self.model
+        encoded = super(CogTemplate, self)._encode(inputs)
+        videos_path = inputs.videos or []
+        video = load_batch(videos_path, load_video_cogvlm2)
+        history_inputs = inputs.to_history()
+        inputs2 = model.build_conversation_input_ids(
+            self.processor,
+            query=history_inputs['query'],
+            history=history_inputs['history'],
+            images=video,
+            template_version='chat')
+        video_token_len = inputs2['token_type_ids'].sum().item()
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        encoded['token_type_ids'] = [0] + [1] * video_token_len + [0] * len(input_ids[1:])
+        encoded['input_ids'] = input_ids[:1] + [self.processor.pad_token_id] * video_token_len + input_ids[1:]
+        if labels is not None:
+            encoded['labels'] = labels[:1] + [-100] * video_token_len + labels[1:]
+        if len(video) > 0:
+            dtype = model.dtype
+            encoded['images'] = [[img.to(dtype=dtype)] for img in inputs2['images']]
+        return encoded
+
+
+register_template(CogVLMTemplateMeta(
+    MLLMTemplateType.cogvlm2_video,
+    template_cls=Cog2VideoTemplate,
+))
+
+
+class GLMEdgeVTemplate(Template):
+    placeholder_tokens = ['<|begin_of_image|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<|begin_of_image|>' * 578]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            encoded['pixel_values'] = torch.tensor(self.processor(images).pixel_values)
+        return encoded
+
+
+register_template(
+    GLM4TemplateMeta(
+        MLLMTemplateType.glm_edge_v,
+        prompt=['<|user|>\\n{{QUERY}}\\n<|assistant|>\\n'],
+        chat_sep=['\\n'],
+        system_prefix=['<|system|>\\n{{SYSTEM}}\\n'],
+        suffix=['<|endoftext|>'],
+        template_cls=GLMEdgeVTemplate,
+    ))

swift/llm/template/template/idefics3.py

@@ -0,0 +1,37 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import align_image_inputs
+
+
+class Idefics3Template(Template):
+    placeholder_tokens = ['<image>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images or []
+        processor = self.processor
+        prompt = self.processor.decode(encoded['input_ids'])
+        if images:
+            image_inputs = processor(text=prompt, images=images, return_tensors='pt', add_special_tokens=False)
+            image_token = 128257  # <image>
+            encoded['input_ids'], encoded['labels'] = align_image_inputs(encoded['input_ids'], encoded['labels'],
+                                                                         image_inputs['input_ids'][0], image_token)
+            encoded['pixel_values'] = image_inputs['pixel_values']
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.idefics3,
+        prefix=['<|begin_of_text|>'],
+        prompt=['User:{{QUERY}}<end_of_utterance>\nAssistant:'],
+        chat_sep=['<end_of_utterance>\n'],
+        suffix=['<end_of_utterance>'],
+        system_prefix=['System:{{SYSTEM}}<end_of_utterance>\n'],
+        template_cls=Idefics3Template,
+    ))

swift/llm/template/template/internlm.py

@@ -0,0 +1,195 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+import torch
+from PIL import Image
+from transformers.dynamic_module_utils import get_class_from_dynamic_module
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType, RMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, Word
+from ..vision_utils import load_file
+from .utils import ChatmlTemplateMeta
+
+INTERNLM_SYSTEM = (
+    'You are an AI assistant whose name is InternLM (书生·浦语).\n'
+    '- InternLM (书生·浦语) is a conversational language model that is developed by Shanghai AI Laboratory (上海人工智能实验室). '
+    'It is designed to be helpful, honest, and harmless.\n'
+    '- InternLM (书生·浦语) can understand and communicate fluently in the language chosen '
+    'by the user such as English and 中文.')
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.internlm,
+        prefix=['<s>'],
+        prompt=['<|User|>:{{QUERY}}\n<|Bot|>:'],
+        chat_sep=['<eoa>\n'],
+        suffix=['<eoa>'],
+        default_system=INTERNLM_SYSTEM,
+        system_prefix=['<s><|System|>:{{SYSTEM}}\n']))
+
+register_template(ChatmlTemplateMeta(LLMTemplateType.internlm2, default_system=INTERNLM_SYSTEM))
+
+register_template(ChatmlTemplateMeta(RMTemplateType.internlm2_reward, suffix=['<|im_end|>\n<|reward|>']))
+
+
+class InternLMXComposer2Template(Template):
+    image_placeholder = ['</s>']
+    version = 'v2'
+    skip_prompt = False
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'video':
+            inputs.images.insert(inputs.image_idx, inputs.videos[index])
+            inputs.image_idx += 1
+        return self.image_placeholder
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        model = self.model
+        encoded = super()._encode(inputs)
+        images = inputs.images or []
+
+        if self.version == 'v2.5':
+            hd_num = 24
+            if len(images) > 1:
+                hd_num = 6
+            hd_num = get_env_args('hd_num', int, hd_num)
+            images_origin = images
+            images = []
+            for image in images_origin:
+                if isinstance(image, Image.Image):
+                    Image_transform = get_class_from_dynamic_module('ixc_utils.Image_transform', model.model_dir)
+                    images.append(Image_transform(image, hd_num=hd_num))
+                else:
+                    load_video = get_class_from_dynamic_module('ixc_utils.load_video', model.model_dir)
+                    frame2img = get_class_from_dynamic_module('ixc_utils.frame2img', model.model_dir)
+                    Video_transform = get_class_from_dynamic_module('ixc_utils.Video_transform', model.model_dir)
+                    image = load_video(load_file(image))
+                    image = frame2img(image, model.font)
+                    images.append(Video_transform(image, hd_num=hd_num))
+        elif self.version == 'v2-4khd':
+            hd_num = 55
+            hd_num = get_env_args('hd_num', int, hd_num)
+            HD_transform = get_class_from_dynamic_module('ixc_utils.HD_transform', model.model_dir)
+            images = [HD_transform(image, hd_num=hd_num) for image in images]
+        images = [model.vis_processor(image).to(model.dtype) for image in images]
+        encoded['images'] = images
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        batch_size = len(inputs['input_ids'])
+        res = []
+        im_mask = []
+        length = inputs['length']
+        for i in range(batch_size):
+            input_ids = inputs['input_ids'][i].tolist()[:length[i]]
+            input_ids.append(2)  # add dummy </s>
+            labels = inputs.get('labels')
+            if labels is not None:
+                labels = labels[i].tolist()[:length[i]]
+                labels.append(2)
+            else:
+                labels = []
+            images = inputs['images'][i]
+            res_inputs_embeds = []
+            res_labels = []
+            wrap_im_mask = []
+            pre_i, i, idx = 0, 0, 0
+            device = model.device
+            internlm2_model = model.model
+            if not hasattr(internlm2_model, 'tok_embeddings'):
+                internlm2_model = internlm2_model.model
+            tok_embeddings = internlm2_model.tok_embeddings
+            if len(images) > 0:
+                images = torch.concat([model.img2emb(image[None])[0] for image in images], dim=0)
+            add_bos = False
+            while i < len(input_ids):
+                if input_ids[i] == 2:  # replace_token
+                    res_input_ids = torch.tensor(([1] if add_bos else []) + input_ids[pre_i:i], device=device)
+                    if not add_bos and self.version != 'v2.5':
+                        add_bos = True
+                    res_inputs_embeds.append(tok_embeddings(res_input_ids[None])[0])
+                    wrap_im_mask += [0] * len(res_input_ids)
+                    res_labels += ([-100] if add_bos else []) + labels[pre_i:i]
+                    if len(images) > 0 and idx < images.shape[0]:
+                        res_inputs_embeds.append(images[idx].to(device))
+                        wrap_im_mask += [1] * images.shape[1]
+                        res_labels += [-100] * images.shape[1]
+                    idx += 1
+                    i += 1
+                    pre_i = i
+                    continue
+                i += 1
+            if len(labels) == 0:
+                res_labels = None
+            im_mask.append(torch.tensor(wrap_im_mask, dtype=torch.bool, device=device))
+            res.append({'inputs_embeds': torch.concat(res_inputs_embeds, dim=0), 'labels': res_labels})
+        res = Template._data_collator(self, res)
+        res['im_mask'] = self._pad_sequence(im_mask, 0)
+        return res
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        res['length'] = [len(b['input_ids']) for b in batch]
+        res.update(self.fetch_inputs(batch, ['images']))
+        return res
+
+
+@dataclass
+class Xcomposer2TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<s>'])
+    prompt: Prompt = field(
+        default_factory=lambda: ['[UNUSED_TOKEN_146]user\n{{QUERY}}[UNUSED_TOKEN_145]\n[UNUSED_TOKEN_146]assistant\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['[UNUSED_TOKEN_145]\n'])
+    suffix: Prompt = field(default_factory=lambda: ['[UNUSED_TOKEN_145]'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<s>[UNUSED_TOKEN_146]system\n{{SYSTEM}}[UNUSED_TOKEN_145]\n'])
+    stop_words: List[Word] = field(default_factory=lambda: ['<|im_end|>'])
+
+
+register_template(
+    Xcomposer2TemplateMeta(
+        MLLMTemplateType.xcomposer2,
+        template_cls=InternLMXComposer2Template,
+        default_system=('You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔).\n'
+                        '- InternLM-XComposer (浦语·灵笔) is a conversational language model that is developed by '
+                        'Shanghai AI Laboratory (上海人工智能实验室). '
+                        'It is designed to be helpful, honest, and harmless.\n'
+                        '- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language chosen '
+                        'by the user such as English and 中文.'),
+    ))
+
+
+class InternLMXComposer2_5Template(InternLMXComposer2Template):
+    system = ('You are an AI assistant whose name is InternLM-XComposer (浦语·灵笔).\n'
+              '- InternLM-XComposer (浦语·灵笔) is a multi-modality conversational language model '
+              'that is developed by Shanghai AI Laboratory (上海人工智能实验室). '
+              'It is designed to be helpful, honest, and harmless.\n'
+              '- InternLM-XComposer (浦语·灵笔) can understand and communicate fluently in the language chosen '
+              'by the user such as English and 中文.\n'
+              '- InternLM-XComposer (浦语·灵笔) is capable of comprehending and articulating responses effectively '
+              'based on the provided image.')
+    version = 'v2.5'
+
+
+class InternLMXComposer2_4khdTemplate(InternLMXComposer2Template):
+    version = 'v2-4khd'
+
+
+register_template(
+    Xcomposer2TemplateMeta(
+        MLLMTemplateType.xcomposer2_5,
+        template_cls=InternLMXComposer2_5Template,
+        default_system=InternLMXComposer2_5Template.system))
+
+register_template(
+    Xcomposer2TemplateMeta(
+        MLLMTemplateType.xcomposer2_4khd,
+        template_cls=InternLMXComposer2_4khdTemplate,
+        default_system=InternLMXComposer2_5Template.system))

swift/llm/template/template/internvl.py

@@ -0,0 +1,168 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from functools import partial
+from typing import Any, Dict, List, Literal
+
+import torch
+from torch import nn
+
+from swift.utils import get_env_args, is_deepspeed_enabled
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, findall
+from ..vision_utils import load_video_internvl, transform_image
+from .microsoft import Phi3TemplateMeta
+from .utils import ChatmlTemplateMeta
+
+
+class InternvlTemplate(Template):
+    skip_prompt = False
+    num_image_token = 256
+    placeholder_tokens = ['<IMG_CONTEXT>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if self.mode == 'vllm':
+            image_context = ['<image>\n']
+        else:
+            image_context = ['<img>', [-100], '</img>\n']
+        return image_context
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        idx_list = findall(input_ids, -100)
+        pixel_values = None
+        images = inputs.images
+        if images:
+            labels = encoded.get('labels')
+            input_size = get_env_args('input_size', int, 448)
+            max_num = get_env_args('max_num', int, 12)
+            pixel_values_images = [transform_image(image, input_size, max_num) for image in images]
+            pixel_values = torch.cat(pixel_values_images, dim=0).to(self.model_info.torch_dtype)
+            image_bs = pixel_values.shape[0]
+
+            idx, idx2 = idx_list[0], idx_list[-1]  # remove [-100, -100]
+            img_tokens: List[int] = self.processor.encode(
+                '<IMG_CONTEXT>', add_special_tokens=False) * self.num_image_token * image_bs
+            input_ids = input_ids[:idx] + img_tokens + input_ids[idx2 + 1:]
+            if labels is not None:
+                labels = labels[:idx] + [-100] * len(img_tokens) + labels[idx2 + 1:]
+            encoded['input_ids'] = input_ids
+            encoded['labels'] = labels
+        encoded['pixel_values'] = pixel_values
+        return encoded
+
+    def compute_loss_context(self, model, inputs):
+        model_name = model.language_model.__class__.__name__.lower()
+        if self._packing and 'internlm2' in model_name:
+            position_ids = inputs['position_ids']
+            modeling_module = model.language_model.model.layers[0].attention.__class__
+            return self._patch_flash_attention_forward(modeling_module, position_ids, use_new_func=True)
+        else:
+            return super().compute_loss_context(model, inputs)
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        embedding = model.get_input_embeddings()
+        device = embedding.weight.device
+        input_ids = inputs['input_ids']
+        inputs_embeds = embedding(input_ids).to(device=device)
+        pixel_values = inputs.get('pixel_values')
+        if pixel_values is not None:
+            pixel_values = pixel_values.to(device=device)
+            vit_embeds = model.extract_feature(pixel_values).to(device=device)
+            selected = (input_ids == self.processor.encode('<IMG_CONTEXT>', add_special_tokens=False)[0])
+            inputs_embeds[selected] = vit_embeds.reshape(-1, vit_embeds.shape[-1])
+        elif is_deepspeed_enabled():
+            dummy_pixel_values = torch.zeros((1, 3, 32, 32), device=device, dtype=inputs_embeds.dtype)
+            vit_embeds = model.extract_feature(dummy_pixel_values).to(device=device)
+            inputs_embeds += vit_embeds.mean() * 0.
+        return {'inputs_embeds': inputs_embeds}
+
+
+register_template(
+    ChatmlTemplateMeta(
+        MLLMTemplateType.internvl,
+        default_system='You are an AI assistant whose name is InternLM (书生·浦语).',
+        template_cls=InternvlTemplate,
+        auto_add_bos=True))
+register_template(
+    Phi3TemplateMeta(
+        MLLMTemplateType.internvl_phi3,
+        default_system='You are an AI assistant whose name is Phi-3.',
+        template_cls=InternvlTemplate,
+        auto_add_bos=True))
+
+
+class Internvl2Template(InternvlTemplate):
+    video_segments = 8
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        image_context = super().replace_tag('image', index, inputs)
+        if media_type == 'image':
+            return image_context
+        elif media_type == 'video':
+            video_segments = get_env_args('video_segments', int, self.video_segments)
+            load_video = partial(load_video_internvl, num_segments=video_segments)
+            return self.replace_video2image(load_video, inputs, lambda i: [f'Frame{i + 1}: '] + image_context)
+
+    def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [f'<ref>{ref}</ref>']
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [f'<box>[{bbox}]</box>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super(InternvlTemplate, self)._encode(inputs)
+        input_ids = encoded['input_ids']
+        idx_list = findall(input_ids, -100)
+        labels = encoded['labels']
+        images = inputs.images
+        if images:
+            has_video = bool(inputs.videos)
+            input_size = get_env_args('input_size', int, 448)
+            max_num = get_env_args('max_num', int, 12)
+            video_max_num = get_env_args('video_max_num', int, 1)
+            if has_video:
+                max_num = video_max_num
+            pixel_values = [transform_image(image, input_size, max_num) for image in images]
+            num_patches = [pv.shape[0] for pv in pixel_values]
+            pixel_values = torch.cat(pixel_values).to(self.model_info.torch_dtype)
+        else:
+            pixel_values = None
+            num_patches = []
+        assert len(num_patches) == len(
+            idx_list), f'len(num_patches): {len(num_patches)}, len(idx_list): {len(idx_list)}'
+
+        def _get_new_tokens(i):
+            img_tokens: List[int] = self.processor.encode(
+                '<IMG_CONTEXT>', add_special_tokens=False) * self.num_image_token * num_patches[i]
+            return img_tokens
+
+        encoded['input_ids'], encoded['labels'] = self._extend_tokens(input_ids, labels, idx_list, _get_new_tokens)
+        encoded['pixel_values'] = pixel_values
+        return encoded
+
+
+_internvl2_system = '你是由上海人工智能实验室联合商汤科技开发的书生多模态大模型，英文名叫InternVL, 是一个有用无害的人工智能助手。'
+register_template(
+    ChatmlTemplateMeta(
+        MLLMTemplateType.internvl2,
+        default_system=_internvl2_system,
+        template_cls=Internvl2Template,
+    ))
+
+register_template(
+    Phi3TemplateMeta(
+        MLLMTemplateType.internvl2_phi3,
+        default_system=_internvl2_system,
+        template_cls=Internvl2Template,
+    ))
+
+register_template(
+    ChatmlTemplateMeta(
+        MLLMTemplateType.internvl2_5,
+        template_cls=Internvl2Template,
+        default_system='你是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。'))

swift/llm/template/template/llama.py

@@ -0,0 +1,213 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import datetime as dt
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+import torch
+import torch.nn as nn
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, Word, findall
+from ..vision_utils import load_batch
+
+# ref: https://github.com/facebookresearch/llama/blob/main/llama/generation.py
+LLAMA_DEFAULT_SYSTEM = (
+    'You are a helpful, respectful and honest assistant. '
+    'Always answer as helpfully as possible, while being safe. '
+    'Your answers should not include any harmful, unethical, racist, sexist, toxic, dangerous, or illegal content. '
+    'Please ensure that your responses are socially unbiased and positive in nature.\n\n'
+    'If a question does not make any sense, or is not factually coherent, '
+    'explain why instead of answering something not correct. '
+    "If you don't know the answer to a question, please don't share false information.")
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.llama, ['<s>[INST] '], ['{{QUERY}} [/INST]'], ['</s><s>[INST] '], ['</s>'],
+        default_system=LLAMA_DEFAULT_SYSTEM,
+        system_prefix=['<s>[INST] <<SYS>>\n{{SYSTEM}}\n<</SYS>>\n\n']))
+
+
+@dataclass
+class Llama3TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<|begin_of_text|>'])
+    prompt: Prompt = field(default_factory=lambda: [
+        '<|start_header_id|>user<|end_header_id|>\n\n{{QUERY}}<|eot_id|>'
+        '<|start_header_id|>assistant<|end_header_id|>\n\n'
+    ])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|eot_id|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|eot_id|>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{{SYSTEM}}<|eot_id|>'])
+    agent_template: str = 'llama3'
+
+
+register_template(Llama3TemplateMeta(LLMTemplateType.llama3))
+
+
+def _get_llama3_2_prefix() -> Prompt:
+    now = dt.datetime.now()
+    date_string = now.strftime('%d %b %Y')
+    date_prompt = f'Cutting Knowledge Date: December 2023\nToday Date: {date_string}'
+    return [f'<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\n{date_prompt}\n\n' '{{SYSTEM}}<|eot_id|>']
+
+
+@dataclass
+class Llama3_2TemplateMeta(Llama3TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: _get_llama3_2_prefix())
+    system_prefix: Optional[Prompt] = None
+
+
+register_template(Llama3_2TemplateMeta(LLMTemplateType.llama3_2))
+
+
+class Llama3_2VisionTemplate(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<|image|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from transformers.models.mllama.processing_mllama import (get_cross_attention_token_mask,
+                                                                  convert_sparse_cross_attention_mask_to_dense)
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            input_ids = encoded['input_ids']
+            processor = self.processor
+            image_features = processor.image_processor(images, return_tensors='pt')
+            num_tiles = image_features.pop('num_tiles')
+            encoded.update(image_features)
+
+            cross_attention_token_mask = [get_cross_attention_token_mask(input_ids, processor.image_token_id)]
+            cross_attention_mask = convert_sparse_cross_attention_mask_to_dense(
+                cross_attention_token_mask,
+                num_tiles=num_tiles,
+                max_num_tiles=processor.image_processor.max_image_tiles,
+                length=len(input_ids),
+            )
+            encoded['cross_attention_mask'] = torch.tensor(cross_attention_mask)
+
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        for key in ['aspect_ratio_ids', 'aspect_ratio_mask']:
+            value = [b[key] for b in batch if b.get(key) is not None]
+            if value:
+                res[key] = torch.concat(value)
+
+        cross_attention_mask = [
+            b['cross_attention_mask'][0] for b in batch if b.get('cross_attention_mask') is not None
+        ]
+        if cross_attention_mask:
+            res['cross_attention_mask'] = self._pad_sequence(cross_attention_mask, 0)
+        return res
+
+
+register_template(Llama3_2TemplateMeta(MLLMTemplateType.llama3_2_vision, template_cls=Llama3_2VisionTemplate))
+
+
+class Llama4Template(Template):
+    placeholder_tokens = ['<|patch|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return [[-100]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            split_token = self._tokenize('\n')
+            input_ids, labels = encoded['input_ids'], encoded['labels']
+            idx_list = findall(input_ids, -100)
+            media_inputs = self.processor(
+                text='\n'.join(['<|image|>'] * len(idx_list)),
+                images=images,
+                add_special_tokens=False,
+                return_tensors='pt')
+            splited_tokens = self._split_list(media_inputs['input_ids'][0].tolist(), split_token)
+
+            encoded['input_ids'], encoded['labels'] = self._extend_tokens(input_ids, labels, idx_list,
+                                                                          lambda i: splited_tokens[i])
+            encoded['pixel_values'] = media_inputs['pixel_values']
+        return encoded
+
+
+@dataclass
+class Llama4TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<|begin_of_text|>'])
+    prompt: Prompt = field(
+        default_factory=lambda:
+        ['<|header_start|>user<|header_end|>\n\n{{QUERY}}<|eot|>'
+         '<|header_start|>assistant<|header_end|>\n\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|eot|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|eot|>'])
+    stop_words: List[Word] = field(default_factory=lambda: ['<|end_of_text|>', '<|eom|>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<|begin_of_text|><|header_start|>system<|header_end|>\n\n{{SYSTEM}}<|eot|>'])
+    agent_template: str = 'llama4'
+
+
+register_template(Llama4TemplateMeta(MLLMTemplateType.llama4, template_cls=Llama4Template))
+
+register_template(
+    Llama3TemplateMeta(
+        LLMTemplateType.reflection,
+        default_system=('You are a world-class AI system, capable of complex reasoning and reflection. '
+                        'Reason through the query inside <thinking> tags, and then provide your final '
+                        'response inside <output> tags. If you detect that you made a mistake in your reasoning '
+                        'at any point, correct yourself inside <reflection> tags.')))
+
+
+class Llama3_1OmniTemplate(Template):
+    skip_prompt = False
+    audio_placeholder = [[-200]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        import whisper
+        encoded = super()._encode(inputs)
+        audios = inputs.audios
+        if audios:
+            audios = load_batch(audios, whisper.load_audio)
+            n_mels = get_env_args('n_mels', int, 128)
+            for i, audio in enumerate(audios):
+                audio = whisper.pad_or_trim(audio)
+                audios[i] = whisper.log_mel_spectrogram(audio, n_mels=n_mels).permute(1, 0)
+            audios = torch.stack(audios)
+            encoded.update({'speech': audios, 'speech_lengths': torch.tensor([[audios.shape[1]]])})
+
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        speech = inputs.get('speech')
+        input_ids = inputs['input_ids']
+        labels = inputs.get('labels')
+        if speech is not None:
+            speech_lengths = inputs['speech_lengths']
+            speech = speech.to(model.dtype)
+            inputs_embeds, labels = model.prepare_inputs_labels_for_speech_and_text(input_ids, None, None, None, labels,
+                                                                                    speech, speech_lengths)[4:]
+        else:
+            inputs_embeds = model.get_model().embed_tokens(input_ids)
+        res = {'inputs_embeds': inputs_embeds}
+        if labels is not None:
+            res['labels'] = labels[0]
+        return res
+
+
+register_template(
+    Llama3TemplateMeta(
+        MLLMTemplateType.llama3_1_omni,
+        default_system=('You are a helpful language and speech assistant. '
+                        'You are able to understand the speech content that the user provides, '
+                        'and assist the user with a variety of tasks using natural language.'),
+        template_cls=Llama3_1OmniTemplate,
+    ))

swift/llm/template/template/llava.py

@@ -0,0 +1,309 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+import torch
+import transformers
+from packaging import version
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+from ..vision_utils import load_video_llava
+from .llama import Llama3TemplateMeta
+from .qwen import QwenTemplateMeta
+from .utils import ChatmlTemplateMeta
+
+
+class LlavaHfTemplate(Template):
+    placeholder_tokens = ['<image>']
+
+    @property
+    def image_token_index(self):
+        if not hasattr(self, '_image_token_index'):
+            self._image_token_index = self.tokenizer.convert_tokens_to_ids(self.processor.image_token)
+        return self._image_token_index
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['<image>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            image_processor = self.processor.image_processor
+            image_inputs = image_processor(images, return_tensors='pt').to(self.model_info.torch_dtype)
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            if 'image_sizes' in image_inputs:
+                encoded['image_sizes'] = image_inputs['image_sizes']
+            if version.parse(transformers.__version__) >= version.parse('4.47'):
+                input_ids = encoded['input_ids']
+                labels = encoded['labels']
+                idx_list = findall(input_ids, self.image_token_index)  # <image>
+                height, width = image_inputs['pixel_values'][0].shape[-2:]
+                added_tokens_len = 0
+                for i, idx in enumerate(idx_list):
+                    if 'image_sizes' in image_inputs:
+                        orig_height, orig_width = image_inputs['image_sizes'][i].tolist()
+                        num_image_tokens = self.processor._get_number_of_features(orig_height, orig_width, height,
+                                                                                  width)
+                    else:
+                        num_image_tokens = (height // self.processor.patch_size) * (
+                            width // self.processor.patch_size) + self.processor.num_additional_image_tokens
+                    if self.processor.vision_feature_select_strategy == 'default':
+                        num_image_tokens -= 1
+                    input_ids = input_ids[:added_tokens_len + idx] + [self.image_token_index] * num_image_tokens \
+                        + input_ids[added_tokens_len + idx + 1:]
+                    if labels is not None:
+                        labels = labels[:added_tokens_len + idx] + [-100] * num_image_tokens \
+                            + labels[added_tokens_len + idx + 1:]
+                    added_tokens_len += num_image_tokens - 1
+                encoded['input_ids'] = input_ids
+                encoded['labels'] = labels
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.llava1_5_hf,
+        prefix=['<s>'],
+        prompt=['USER: {{QUERY}}\nASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        system_prefix=['<s>{{SYSTEM}}\n'],
+        template_cls=LlavaHfTemplate,
+    ))
+
+
+class LlavaVideoHfTemplate(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return ['<image>\n']
+        assert media_type == 'video'
+        media_file = inputs.videos[index]
+        if media_file.rsplit('.', 1)[-1] in {'jpg', 'png'}:
+            return ['<image>\n']
+        else:
+            inputs.videos[index] = load_video_llava(inputs.videos[index])
+            return ['<video>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images or []
+        videos = inputs.videos or []
+        if len(videos) > 0:
+            video_processor = self.processor.video_processor
+            video_inputs = video_processor(videos, return_tensors='pt').to(self.model_info.torch_dtype)
+            encoded['pixel_values_videos'] = video_inputs['pixel_values_videos']
+        if len(images) > 0:
+            image_processor = self.processor.image_processor
+            image_inputs = image_processor(images, return_tensors='pt').to(self.model_info.torch_dtype)
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            encoded['image_sizes'] = image_inputs['image_sizes']
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.llava_next_video_hf,
+        prefix=['{{SYSTEM}} '],
+        prompt=['USER: {{QUERY}} ASSISTANT:'],
+        chat_sep=[' '],
+        suffix=[['eos_token_id']],
+        template_cls=LlavaVideoHfTemplate,
+        auto_add_bos=True,
+    ))
+
+
+class Llava1_6HfTemplate(LlavaHfTemplate):
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        for b in batch:
+            pixel_values = b.get('pixel_values')
+            if pixel_values is not None:
+                b['pixel_values'] = pixel_values.squeeze(0)  # 5d -> 4d
+        res = super()._data_collator(batch, padding_to=padding_to)
+        return res
+
+
+@dataclass
+class LlavaMistralTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<s>[INST] '])
+    prompt: Prompt = field(default_factory=lambda: ['{{QUERY}} [/INST]'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['</s>[INST] '])
+    suffix: Prompt = field(default_factory=lambda: ['</s>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<<SYS>>\n{{system}}\n<</SYS>>\n\n'])
+
+
+register_template(LlavaMistralTemplateMeta(MLLMTemplateType.llava1_6_mistral_hf, template_cls=Llava1_6HfTemplate))
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.llava1_6_vicuna_hf,
+        prefix=['<s>'],
+        prompt=['USER: {{QUERY}} ASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        default_system=('A chat between a curious human and an artificial intelligence assistant. '
+                        "The assistant gives helpful, detailed, and polite answers to the human's questions."),
+        system_prefix=['<s>{{SYSTEM}} '],
+        template_cls=Llava1_6HfTemplate))
+
+
+class LLava1_6YiHfTemplate(Llava1_6HfTemplate):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if self.mode == 'vllm':
+            return [[64000], '\n']
+        else:
+            return super().replace_tag(media_type, index, inputs)
+
+
+register_template(ChatmlTemplateMeta(
+    MLLMTemplateType.llava1_6_yi_hf,
+    template_cls=LLava1_6YiHfTemplate,
+))
+
+register_template(Llama3TemplateMeta(
+    MLLMTemplateType.llama3_llava_next_hf,
+    template_cls=Llava1_6HfTemplate,
+))
+
+register_template(QwenTemplateMeta(MLLMTemplateType.llava_next_qwen_hf, template_cls=Llava1_6HfTemplate))
+
+
+class LlavaOneVisionHfTemplate(Llava1_6HfTemplate):
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = Template._encode(self, inputs)
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, 151646)  # <image>
+        processor = self.processor
+        if images:
+            image_processor = processor.image_processor
+            image_inputs = image_processor(images, return_tensors='pt').to(self.model_info.torch_dtype)
+            height, width = image_inputs['pixel_values'][0].shape[-2:]
+            added_tokens_len = 0
+            for idx, pixel_v, image_size in zip(idx_list, image_inputs['pixel_values'], image_inputs['image_sizes']):
+                if isinstance(image_size, torch.Tensor):
+                    image_size = image_size.tolist()
+                orig_height, orig_width = image_size
+                num_image_tokens = processor._get_number_of_features(orig_height, orig_width, height, width)
+                input_ids = input_ids[:added_tokens_len
+                                      + idx] + [151646] * num_image_tokens + input_ids[added_tokens_len + idx + 1:]
+                if labels is not None:
+                    labels = labels[:added_tokens_len + idx] + [-100] * num_image_tokens + labels[added_tokens_len + idx
+                                                                                                  + 1:]
+                added_tokens_len += num_image_tokens - 1
+            encoded['input_ids'] = input_ids
+            encoded['labels'] = labels
+            encoded['pixel_values'] = image_inputs['pixel_values']
+            if 'image_sizes' in image_inputs:
+                encoded['image_sizes'] = image_inputs['image_sizes']
+        return encoded
+
+
+register_template(
+    QwenTemplateMeta(
+        MLLMTemplateType.llava_onevision_hf,
+        default_system=None,
+        template_cls=LlavaOneVisionHfTemplate,
+    ))
+
+
+class LlavaLlama3_1HfTemplate(LlavaHfTemplate):
+    # DaozeZhang
+    system = ('You are a helpful language and vision assistant. '
+              'You are able to understand the visual content that the user provides, '
+              'and assist the user with a variety of tasks using natural language.')
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        if len(encoded['pixel_values'].shape) == 5:  # (1, num_patch, 3, H/W, W/H)
+            encoded['pixel_values'] = torch.squeeze(encoded['pixel_values'], dim=0)  # (num_patch, 3, H/W, W/H)
+        return encoded
+
+
+register_template(
+    Llama3TemplateMeta(
+        MLLMTemplateType.llava_llama3_1_hf,
+        default_system=LlavaLlama3_1HfTemplate.system,
+        template_cls=LlavaLlama3_1HfTemplate,
+    ))
+
+
+class LLavaLlama3HfTemplate(Template):
+    # xtuner
+    image_placeholder = ['<image>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        raw_image = inputs.images
+        if raw_image:
+            pixel_values = self.processor.image_processor(raw_image, return_tensors='pt')['pixel_values']
+            encoded['pixel_values'] = pixel_values.to(self.model_info.torch_dtype)
+        return encoded
+
+
+register_template(Llama3TemplateMeta(
+    MLLMTemplateType.llava_llama3_hf,
+    template_cls=LLavaLlama3HfTemplate,
+))
+
+
+class LLavaTemplate(Template):
+    skip_prompt = False
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return [[-200], '\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images or []
+        image_sizes = [x.size for x in images]
+        from llava.mm_utils import process_images
+        model = self.model.model
+        if not hasattr(model, 'vision_tower'):
+            model = model.model
+        image_processor = model.vision_tower.image_processor
+        if images:
+            images_tensor = process_images(images, image_processor, model.config)
+            encoded['images'] = images_tensor.to(model.dtype).squeeze(0)
+            encoded['image_sizes'] = image_sizes
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = [b['images'] for b in batch if 'images' in b]
+        if images:
+            res['images'] = images
+            res['image_sizes'] = sum([b['image_sizes'] for b in batch if 'image_sizes' in b], start=[])
+        return res
+
+
+register_template(LlavaMistralTemplateMeta(MLLMTemplateType.llava1_6_mistral, template_cls=LLavaTemplate))
+
+register_template(ChatmlTemplateMeta(MLLMTemplateType.llava1_6_yi, template_cls=LLavaTemplate))
+
+register_template(
+    Llama3TemplateMeta(
+        MLLMTemplateType.llama3_llava_next,
+        template_cls=LLavaTemplate,
+        default_system=('You are a helpful language and vision assistant. '
+                        'You are able to understand the visual content that the user provides, '
+                        'and assist the user with a variety of tasks using natural language.'),
+    ))
+
+register_template(QwenTemplateMeta(MLLMTemplateType.llava_next_qwen, template_cls=LLavaTemplate))

swift/llm/template/template/llm.py

@@ -0,0 +1,274 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from datetime import datetime
+from typing import Optional
+
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..utils import Prompt
+from .llama import Llama3_2TemplateMeta
+from .qwen import Qwen2VLTemplate, QwenTemplateMeta
+from .utils import DEFAULT_SYSTEM, ChatmlTemplateMeta
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.default,
+        prefix=[],
+        prompt=['### Human:\n{{QUERY}}\n\n### Assistant:\n'],
+        chat_sep=['\n\n'],
+        default_system=DEFAULT_SYSTEM,
+        system_prefix=['{{SYSTEM}}\n\n'],
+        auto_add_bos=True))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.modelscope_agent,
+        prefix=[],
+        prompt=[' \n\n<|user|>:{{QUERY}} \n\n<|assistant|>:'],
+        chat_sep=[],
+        suffix=[' \n\n</s>'],
+        system_prefix=[' \n\n<|system|>:{{SYSTEM}}'],
+        default_system=DEFAULT_SYSTEM,
+    ))
+
+register_template(QwenTemplateMeta(MLLMTemplateType.qwen2_gme, template_cls=Qwen2VLTemplate, suffix=['<|endoftext|>']))
+
+register_template(
+    TemplateMeta(LLMTemplateType.baichuan, prefix=['{{SYSTEM}}'], prompt=[[195], '{{QUERY}}', [196]], chat_sep=[]))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.baichuan_m1,
+        prefix=[],
+        prompt=['<C_Q>{{QUERY}}<C_A>'],
+        chat_sep=[],
+        suffix=['<C_A>'],
+        system_prefix=['<B_SYS>{{SYSTEM}}'],
+        default_system=DEFAULT_SYSTEM,
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.numina,
+        prefix=[['bos_token_id']],
+        prompt=['### Problem: {{QUERY}}\n### Solution: '],
+        chat_sep=['\n'],
+        system_prefix=[['bos_token_id'], '{{SYSTEM}}']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.mistral_nemo,
+        prefix=['<s>[INST] '],
+        prompt=['{{SYSTEM}}\n\n', '{{QUERY}}[/INST]'],
+        chat_sep=['</s>[INST] '],
+        suffix=['</s>']))
+
+today = datetime.now().strftime('%Y-%m-%d')
+
+mistral_2501_system = (
+    'You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup '
+    'headquartered in Paris.\n'
+    f'Your knowledge base was last updated on 2023-10-01. The current date is {today}.\n\n'
+    "When you're not sure about some information, you say that you don't have the information and don't "
+    'make up anything.\n'
+    "If the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer "
+    'the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. '
+    '"What are some good restaurants around me?" => "Where are you?" or "When is the next flight to Tokyo" => "'
+    'Where do you travel from?")')
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.mistral_2501,
+        prefix=['<s>'],
+        prompt=['[INST]{{QUERY}}[/INST]'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        system_prefix=['<s>[SYSTEM_PROMPT]{{SYSTEM}}[/SYSTEM_PROMPT]'],
+        default_system=mistral_2501_system))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.xverse,
+        prefix=['{{SYSTEM}}'],
+        prompt=['Human: {{QUERY}}\n\nAssistant: '],
+        chat_sep=[['eos_token_id']]))
+
+register_template(TemplateMeta(LLMTemplateType.yuan, prefix=[], prompt=['{{QUERY}}<sep>'], chat_sep=None))
+register_template(
+    TemplateMeta(
+        LLMTemplateType.ziya,
+        prefix=[['bos_token_id'], '{{SYSTEM}}'],
+        prompt=['<human>:{{QUERY}}\n<bot>:'],
+        chat_sep=['\n']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.skywork,
+        prefix=['<s>{{SYSTEM}}'],
+        prompt=['</s><s>[USER]{{QUERY}}[SEP][BOT]'],
+        chat_sep=None,
+        suffix=['[SEP]</s>']))
+
+register_template(
+    Llama3_2TemplateMeta(
+        LLMTemplateType.skywork_o1,
+        default_system=(
+            'You are Skywork-o1, a thinking model developed by Skywork AI, specializing in solving complex problems '
+            "involving mathematics, coding, and logical reasoning through deep thought. When faced with a user's "
+            'request, you first engage in a lengthy and in-depth thinking process to explore possible solutions to '
+            'the problem. After completing your thoughts, you then provide a detailed explanation of the solution '
+            'process in your response.'),
+    ))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.bluelm,
+        prefix=[['bos_token_id'], '{{SYSTEM}}'],
+        prompt=['[|Human|]:{{QUERY}}[|AI|]:'],
+        chat_sep=[]))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.codefuse_codellama,
+        prefix=['{{SYSTEM}}'],
+        prompt=['<|role_start|>human<|role_end|>{{QUERY}}<|role_start|>bot<|role_end|>'],
+        chat_sep=[]))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.codefuse,
+        prefix=[],
+        prompt=['<s>human\n{{QUERY}}\n<s>bot\n'],
+        chat_sep=[['eos_token_id'], '\n'],
+        system_prefix=['<s>system\n{{SYSTEM}}\n']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.zephyr,
+        prefix=[],
+        prompt=['<|user|>\n{{QUERY}}</s>\n<|assistant|>\n'],
+        chat_sep=['</s>\n'],
+        suffix=['</s>'],
+        system_prefix=['<|system|>\n{{SYSTEM}}</s>\n']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.sus,
+        prefix=['{{SYSTEM}}'],
+        prompt=['### Human: {{QUERY}}\n\n### Assistant: '],
+        chat_sep=['<|endoftext|>'],
+        suffix=['<|endoftext|>']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.orion,
+        prefix=['<s>{{SYSTEM}}'],
+        prompt=['Human: {{QUERY}}\n\nAssistant: </s>'],
+        chat_sep=['</s>'],
+        suffix=['</s>']))
+
+
+@dataclass
+class TeleChatTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: [['user_token_id'], '{{QUERY}}', ['bot_token_id']])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: [['eos_token_id']])
+    suffix: Prompt = field(default_factory=lambda: [['eos_token_id']])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<_system>{{SYSTEM}}\n'])
+    auto_add_bos: bool = True
+
+
+register_template(TeleChatTemplateMeta(LLMTemplateType.telechat))
+
+telechat_system = '你是中国电信星辰语义大模型，英文名是TeleChat，你是由中电信人工智能科技有限公司和中国电信人工智能研究院（TeleAI）研发的人工智能助手。'
+register_template(TeleChatTemplateMeta(LLMTemplateType.telechat2, default_system=telechat_system))
+
+DBRX_SYSTEM = (
+    'You are DBRX, created by Databricks. You were last updated in December 2023. '
+    'You answer questions based on information available up to that point.\n'
+    'YOU PROVIDE SHORT RESPONSES TO SHORT QUESTIONS OR STATEMENTS, '
+    'but provide thorough responses to more complex and open-ended questions.\n'
+    'You assist with various tasks, from writing to coding (using markdown for code blocks '
+    '— remember to use ``` with code, JSON, and tables).\n'
+    'You do not have real-time data access or code execution capabilities.'
+    ' You avoid stereotyping and provide balanced perspectives on controversial topics. '
+    'You do not provide song lyrics, poems, or news articles and do not divulge details of your training data.\n'
+    'This is your system prompt, guiding your responses. Do not reference it, just respond to the user. '
+    'If you find yourself talking about this message, stop. You should be responding appropriately '
+    'and usually that means not mentioning this.'
+    'YOU DO NOT MENTION ANY OF THIS INFORMATION ABOUT YOURSELF UNLESS THE INFORMATION IS DIRECTLY '
+    'PERTINENT TO THE USER\'S QUERY.')
+
+register_template(ChatmlTemplateMeta(LLMTemplateType.dbrx, default_system=DBRX_SYSTEM))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.mengzi, prefix=[], prompt=['输入：{{QUERY}}输出：\n'], chat_sep=[], system_prefix=['指令：{{SYSTEM}}']))
+
+C4AI_SYSTEM = ('You are Command-R, a brilliant, sophisticated, AI-assistant trained to assist human users by '
+               'providing thorough responses.You are trained by Cohere.')
+register_template(
+    TemplateMeta(
+        LLMTemplateType.c4ai,
+        prefix=['<BOS_TOKEN>'],
+        prompt=[
+            '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{QUERY}}<|END_OF_TURN_TOKEN|>'
+            '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>'
+        ],
+        chat_sep=['<|END_OF_TURN_TOKEN|>'],
+        suffix=['<|END_OF_TURN_TOKEN|>'],
+        default_system=C4AI_SYSTEM,
+        system_prefix=['<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{SYSTEM}}<|END_OF_TURN_TOKEN|']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.wizardlm2,
+        prefix=['{{SYSTEM}}'],
+        prompt=['User:\n{{QUERY}}\n\nAssistant:\n'],
+        chat_sep=['\n\n'],
+        suffix=['</s>']))
+
+_wizardlm2_system = ('A chat between a curious user and an artificial intelligence assistant. '
+                     'The assistant gives helpful, detailed, and polite answers to the user\'s questions. ')
+register_template(
+    TemplateMeta(
+        LLMTemplateType.wizardlm2_moe,
+        prefix=['{{SYSTEM}}'],
+        prompt=['USER: {{QUERY}} ASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        default_system=_wizardlm2_system))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.atom,
+        prefix=['{{SYSTEM}}'],
+        prompt=['<s>Human: {{QUERY}}\n</s><s>Assistant: '],
+        chat_sep=['</s>'],
+        suffix=['</s>']))
+
+AYA_SYSTEM = ('You are Aya, a brilliant, sophisticated, multilingual AI-assistant trained to assist human users by '
+              'providing thorough responses. You are able to interact and respond to questions in 23 languages and '
+              'you are powered by a multilingual model built by Cohere For AI.')
+register_template(
+    TemplateMeta(
+        LLMTemplateType.aya,
+        prefix=['<BOS_TOKEN>'],
+        prompt=[
+            '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{QUERY}}<|END_OF_TURN_TOKEN|>'
+            '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>'
+        ],
+        chat_sep=['<|END_OF_TURN_TOKEN|>'],
+        suffix=['<|END_OF_TURN_TOKEN|>'],
+        default_system=AYA_SYSTEM,
+        system_prefix=['<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{SYSTEM}}<|END_OF_TURN_TOKEN|']))
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.ling,
+        prefix=[],
+        system_prefix=['<role>SYSTEM</role>{{SYSTEM}}'],
+        prompt=['<role>HUMAN</role>{{QUERY}}<role>ASSISTANT</role>'],
+        chat_sep=[],
+        suffix=['<|endoftext|>'],
+    ))

swift/llm/template/template/megrez.py

@@ -0,0 +1,93 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+import torch
+import torch.nn as nn
+
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+
+
+@dataclass
+class MegrezTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<|role_start|>system<|role_end|>{{SYSTEM}}<|turn_end|>'])
+    prompt: Prompt = field(default_factory=lambda:
+                           ['<|role_start|>user<|role_end|>{{QUERY}}<|turn_end|><|role_start|>assistant<|role_end|>'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|turn_end|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|turn_end|>'])
+    default_system: str = '你是Megrez-3B-Instruct，将针对用户的问题给出详细的、积极的回答。'
+
+
+register_template(MegrezTemplateMeta(LLMTemplateType.megrez))
+
+
+class MegrezOmniTemplate(Template):
+    skip_prompt = False
+    placeholder_tokens = ['<|unk|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return [[-1], '\n']
+        elif media_type == 'audio':
+            return [f'Audio {index + 1}: ', [-2], '\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+
+        for mm_key in ['images', 'audios']:
+            mm_data = getattr(inputs, mm_key)
+            if not mm_data:
+                continue
+            if mm_key == 'images':
+                idx_list = findall(input_ids, -1)
+                encoding = self.processor.process_image(
+                    mm_data,
+                    return_tensors='pt',
+                )
+                text = self.processor.insert_image_feature_placeholders(
+                    '<s>'.join(['(<image>./</image>)'] * len(mm_data)), encoding)
+                encoded['image_encoding'] = encoding
+            else:
+                idx_list = findall(input_ids, -2)
+                encoding = self.processor.process_audio(
+                    mm_data,
+                    return_tensors='pt',
+                )
+                text = self.processor.insert_audio_feature_placeholders(
+                    '<s>'.join(['(<audio>./</audio>)'] * len(mm_data)), encoding)
+                encoded['audio_encoding'] = encoding
+
+            padding = text.split('<s>')
+
+            def _get_new_tokens(i):
+                return self._tokenize(padding[i])
+
+            input_ids, labels = self._extend_tokens(input_ids, labels, idx_list, _get_new_tokens)
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        _, inputs_embeds, _ = model.compose_embeddings(inputs)
+        inputs.pop('position_ids', None)
+        return {'inputs_embeds': inputs_embeds}
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        new_batch = []
+        for b in batch:
+            text_encodings = {'input_ids': torch.tensor(b['input_ids'])}
+            multimodal_inputs = {'image_encoding': b.get('image_encoding'), 'audio_encoding': b.get('audio_encoding')}
+            new_batch.append(self.processor.merge_encodings(text_encodings, multimodal_inputs))
+        res.update(self.processor.data_collator(new_batch))
+        return res
+
+
+register_template(MegrezTemplateMeta(MLLMTemplateType.megrez_omni, template_cls=MegrezOmniTemplate))

swift/llm/template/template/microsoft.py

@@ -0,0 +1,205 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+import json
+import torch
+from torch import nn
+
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+from ..vision_utils import load_file
+
+
+class FlorenceTemplate(Template):
+    # If it's an encoder-decoder architecture, the default settings are
+    # loss_scale: 'last_round' and skip_prompt: False.
+    is_encoder_decoder = True
+
+    @staticmethod
+    def _add_default_tags(inputs: StdTemplateInputs) -> None:
+        return
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        return []
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [''.join(f'<loc_{box}>' for box in bbox)]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        processor = self.processor
+        inputs.query = inputs.to_history()['query']
+        new_query = processor._construct_prompts([inputs.query])[0]
+        for i in reversed(range(len(inputs.messages))):
+            if inputs.messages[i]['role'] == 'user':
+                inputs.messages[i]['content'] = new_query
+                break
+        encoded = super()._encode(inputs)
+        input_ids = encoded['prompt_input_ids']
+        images = inputs.images or []
+        labels = encoded['labels']
+        if labels is not None:
+            labels = [0] + labels
+        if images:
+            pixel_values = processor.image_processor(
+                images, return_tensors='pt')['pixel_values'].to(self.model_info.torch_dtype)
+            encoded['pixel_values'] = pixel_values
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        inputs_embeds = model.get_input_embeddings()(inputs['input_ids'])
+        pixel_values = inputs.get('pixel_values')
+        if pixel_values is not None:
+            image_features = model._encode_image(pixel_values)
+            inputs_embeds, inputs['attention_mask'] = model._merge_input_ids_with_image_features(
+                image_features, inputs_embeds)
+        return {'inputs_embeds': inputs_embeds}
+
+    def decode(self, generate_ids: List[int], **kwargs) -> Any:
+        response = super().decode(generate_ids, **kwargs)
+        template_inputs = kwargs.get('template_inputs')
+        images = template_inputs.images
+        image_size = None
+        if images:
+            image_size = (images[0].width, images[0].height)
+        return json.dumps(
+            self.processor.post_process_generation(response, task=template_inputs.query, image_size=image_size))
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.florence,
+        prefix=['<s>'],
+        prompt=['{{QUERY}}</s>'],
+        chat_sep=None,
+        suffix=['</s>'],
+        template_cls=FlorenceTemplate,
+    ))
+
+
+@dataclass
+class Phi3TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: ['<|user|>\n{{QUERY}}<|end|>\n<|assistant|>\n'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|end|>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<|end|>'])
+    system_prefix: Optional[Prompt] = field(default_factory=lambda: ['<|system|>\n{{SYSTEM}}<|end|>\n'])
+    auto_add_bos: bool = True
+
+
+register_template(Phi3TemplateMeta(LLMTemplateType.phi3))
+
+
+@dataclass
+class Phi4TemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(
+        default_factory=lambda: ['<|im_start|>user<|im_sep|>{{QUERY}}<|im_end|><|im_start|>assistant<|im_sep|>'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|im_end|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|im_end|>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<|im_start|>system<|im_sep|>{{SYSTEM}}<|im_end|>'])
+    auto_add_bos: bool = True
+
+
+register_template(Phi4TemplateMeta(LLMTemplateType.phi4))
+
+
+class Phi3VisionTemplate(Template):
+    image_placeholder = ['<|image|><s>\n']  # <|image|>\n
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if self.mode == 'vllm':
+            return [f'<|image_{index + 1}|>\n']  # <|image_1|>\n
+        else:
+            return super().replace_tag(media_type, index, inputs)
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        images = inputs.images or []
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, 32044)  # '<|image|>'
+
+        if len(images) > 0:
+            processor = self.processor
+            encoded.update(processor.image_processor(images, return_tensors='pt'))
+            assert len(idx_list) == len(images), f'len(idx_list): {len(idx_list)}, len(images): {len(images)}'
+            res_input_ids = []
+            res_labels = []
+            num_img_tokens = encoded.pop('num_img_tokens').tolist()
+            idx_list.insert(0, -1)
+            for i in range(len(idx_list) - 1):
+                image_token_id = -i - 1
+                res_input_ids += input_ids[idx_list[i] + 1:idx_list[i + 1]] + [image_token_id] * num_img_tokens[i]
+                if labels is not None:
+                    res_labels += labels[idx_list[i] + 1:idx_list[i + 1]] + [-100] * num_img_tokens[i]
+            res_input_ids += input_ids[idx_list[-1] + 1:]
+            input_ids = res_input_ids
+            if labels is not None:
+                res_labels += labels[idx_list[-1] + 1:]
+                labels = res_labels
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+
+class Phi4MMTemplate(Template):
+    placeholder_tokens = ['<|endoftext10|>', '<|endoftext11|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return [[-100]]
+        elif media_type == 'audio':
+            import soundfile as sf
+            inputs.audios[index] = sf.read(load_file(inputs.audios[index]))
+            return [[-200]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        images_idx = findall(input_ids, -100)
+        audios_idx = findall(input_ids, -200)
+        text = '\n'.join(['<|image_1|>'] * len(inputs.images) + ['<|audio_1|>'] * len(inputs.audios))
+        new_encoded = self.processor(
+            text=text, images=inputs.images or None, audios=inputs.audios or None, return_tensors='pt')
+        placeholders = self._split_list(new_encoded.pop('input_ids')[0].tolist(), 198)
+
+        def _get_new_tokens(i):
+            return placeholders[i]
+
+        encoded['input_ids'], encoded['labels'] = self._extend_tokens(input_ids, labels, images_idx + audios_idx,
+                                                                      _get_new_tokens)
+        new_encoded.pop('attention_mask')
+        encoded.update(new_encoded)
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        keys = [
+            'input_image_embeds', 'image_sizes', 'image_attention_mask', 'input_audio_embeds', 'audio_embed_sizes',
+            'input_mode'
+        ]
+        inputs = self.fetch_inputs(batch, keys)
+        for k, v in inputs.items():
+            inputs[k] = torch.concat(v)
+        res.update(inputs)
+        return res
+
+
+register_template(Phi3TemplateMeta(MLLMTemplateType.phi3_vision, template_cls=Phi3VisionTemplate))
+
+register_template(Phi3TemplateMeta(
+    MLLMTemplateType.phi4_multimodal,
+    template_cls=Phi4MMTemplate,
+))

swift/llm/template/template/minicpm.py

@@ -0,0 +1,229 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from functools import partial
+from typing import Any, Dict, List, Literal, Optional
+
+import torch
+from torch import nn
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+from ..vision_utils import load_video_minicpmv_mplug_owl3
+from .llama import Llama3TemplateMeta
+from .qwen import Qwen2_5TemplateMeta, QwenTemplateMeta
+
+
+@dataclass
+class MinicpmTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<s>{{SYSTEM}}'])
+    prompt: Prompt = field(default_factory=lambda: ['<用户>{{QUERY}}<AI>'])
+    chat_sep: Optional[Prompt] = field(default_factory=list)
+    suffix: Prompt = field(default_factory=lambda: ['</s>'])
+
+
+register_template(MinicpmTemplateMeta(LLMTemplateType.minicpm))
+
+
+def _remove_idx(arr: List[int], idx_list: List[int]) -> List[int]:
+    res = []
+    idx_set = set(idx_list)
+    for i, x in enumerate(arr):
+        if i not in idx_set:
+            res.append(x)
+    return res
+
+
+class MiniCPMVTemplate(Template):
+    is_v2_5 = False
+    use_model = True
+    skip_prompt = False
+    placeholder_tokens = ['<unk>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if self.mode == 'vllm':
+            return ['(<image>./</image>)\n']
+        else:
+            return [[-100]]
+
+    async def prepare_lmdeploy_turbomind_inputs(self, inputs: Dict[str, Any]) -> None:
+        images = inputs.pop('images', None) or []
+        if len(images) == 0:
+            return
+        input_ids = inputs['input_ids']
+        idx_list = findall(input_ids, -100)
+        idx_list.insert(0, -1)
+        new_input_ids = []
+        features = []
+        for i in range(len(idx_list) - 1):
+            new_input_ids += input_ids[idx_list[i] + 1:idx_list[i + 1]]
+            context_list = ['<image>', [-100], '</image>']
+            feat = [x.squeeze() for x in images[i]['embeddings'].split(1)]
+            grid = images[i].get('grid')
+            if len(feat) > 1 and grid is not None:
+                context_list.append('<slice>')
+                for j in range(grid[1]):
+                    if j > 0:
+                        context_list.append('\n')
+                    for _ in range(grid[0]):
+                        context_list += ['<image>', [-100], '</image>']
+                context_list.append('</slice>\n')
+            new_input_ids += self._encode_context_list(context_list)[0]
+            features += feat
+        new_input_ids += input_ids[idx_list[-1] + 1:]
+        inputs['input_ids'] = new_input_ids
+        inputs['images'] = features
+        await super().prepare_lmdeploy_turbomind_inputs(inputs)
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, -100)
+        idx = idx_list[0]
+        tgt_sizes = None
+        slice_mode = getattr(self.config, 'slice_mode', False)
+        if slice_mode:
+            if self.is_v2_5:
+                image_processor = self.processor.image_processor
+                image_inputs = image_processor(images, return_tensors='pt').to(self.model_info.torch_dtype)
+                placeholder = image_processor.get_slice_image_placeholder(image_inputs.image_sizes[0][0])
+                pixel_values = image_inputs['pixel_values']
+                tgt_sizes = image_inputs['tgt_sizes']
+            else:
+                images, placeholder = self.model.get_slice_image_placeholder(images[0], self.processor)
+                pixel_values = [[self.model.transform(img) for img in images]]
+            placeholder += '\n'
+            placeholder_id = self.processor.encode(placeholder, add_special_tokens=False)
+            input_ids = (input_ids[:idx] + placeholder_id + input_ids[idx + 1:])
+            if labels is not None:
+                labels = (labels[:idx] + [-100] * len(placeholder_id) + labels[idx + 1:])
+            input_tensor_ids = torch.tensor(input_ids)
+            image_start_idx = torch.where(input_tensor_ids == self.processor.im_start_id)[0]
+            image_start_idx += 1
+            image_end_idx = torch.where(input_tensor_ids == self.processor.im_end_id)[0]
+            valid_image_nums = max(len(image_start_idx), len(image_end_idx))
+            image_bound = [
+                torch.hstack(
+                    [image_start_idx[:valid_image_nums].unsqueeze(-1), image_end_idx[:valid_image_nums].unsqueeze(-1)])
+            ]
+        else:
+            placeholder = '<image>' + '<unk>' * self.config.query_num + '</image>\n'
+            placeholder_id = self.processor.encode(placeholder, add_special_tokens=False)
+            input_ids = (input_ids[:idx] + placeholder_id + input_ids[idx + 1:])
+            if labels is not None:
+                labels = (labels[:idx] + [-100] * len(placeholder_id) + labels[idx + 1:])
+            image_bound = [torch.tensor([[idx, idx + self.config.query_num]])]
+            pixel_values = [[self.model.transform(images[0])]]
+        encoded = {
+            'input_ids': input_ids,
+            'labels': labels,
+            'image_bound': image_bound,
+            'pixel_values': pixel_values,
+            'tgt_sizes': tgt_sizes
+        }
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        inputs_embeds, _ = model.get_vllm_embedding(inputs)
+        return {'inputs_embeds': inputs_embeds}
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = {}
+        for k in ['pixel_values', 'image_bound', 'tgt_sizes']:
+            res[k] = self.gather_list(batch, k)
+        res.update(super()._data_collator(batch, padding_to=padding_to))
+        return res
+
+
+register_template(MinicpmTemplateMeta(MLLMTemplateType.minicpmv, template_cls=MiniCPMVTemplate))
+
+
+class MiniCPMV2_5Template(MiniCPMVTemplate):
+    is_v2_5 = True
+
+
+register_template(Llama3TemplateMeta(
+    MLLMTemplateType.minicpmv2_5,
+    template_cls=MiniCPMV2_5Template,
+))
+
+
+class MiniCPMV2_6Template(MiniCPMVTemplate):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type in {'image', 'video'}
+        max_num_frames = get_env_args('max_num_frames', int, 64)
+        load_video = partial(load_video_minicpmv_mplug_owl3, max_num_frames=max_num_frames)
+        image_context = super().replace_tag('image', index, inputs)
+        if media_type == 'image':
+            return image_context
+        elif media_type == 'video':
+            return self.replace_video2image(load_video, inputs, lambda i: image_context)
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = Template._encode(self, inputs)
+        images = inputs.images
+        use_video = bool(inputs.videos)
+        use_image_id = True
+        max_slice_nums = get_env_args('max_slice_nums', int, None)
+        video_max_slice_nums = get_env_args('video_max_slice_nums', int, 1)  # or 2
+        if use_video:
+            max_slice_nums = video_max_slice_nums
+            use_image_id = False
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, -100)
+
+        image_processor = self.processor.image_processor
+        image_inputs = image_processor([images], return_tensors='pt',
+                                       max_slice_nums=max_slice_nums).to(self.model_info.torch_dtype)
+
+        def _get_new_tokens(i):
+            placeholder = image_processor.get_slice_image_placeholder(
+                image_inputs.image_sizes[0][i], image_idx=i, max_slice_nums=max_slice_nums, use_image_id=use_image_id)
+            placeholder += '\n'
+            return self.processor.encode(placeholder, add_special_tokens=False)
+
+        input_ids, labels = self._extend_tokens(input_ids, labels, idx_list, _get_new_tokens)
+        if inputs.images:
+            input_tensor_ids = torch.tensor(input_ids)
+            unk_token = self.processor.encode('<unk>', add_special_tokens=False)[0]
+            indices = (input_tensor_ids == unk_token).nonzero(as_tuple=True)[0].tolist()
+
+            ranges = []
+            start = indices[0]
+            for i in range(1, len(indices)):
+                if indices[i] != indices[i - 1] + 1:
+                    ranges.append([start, indices[i - 1] + 1])
+                    start = indices[i]
+            ranges.append([start, indices[-1] + 1])
+            image_bound = [torch.tensor(ranges)]
+        else:
+            image_bound = [[]]
+
+        encoded = {
+            'input_ids': input_ids,
+            'labels': labels,
+            'image_bound': image_bound,
+            'pixel_values': image_inputs['pixel_values'],
+            'tgt_sizes': image_inputs['tgt_sizes']
+        }
+        return encoded
+
+
+register_template(QwenTemplateMeta(
+    MLLMTemplateType.minicpmv2_6,
+    template_cls=MiniCPMV2_6Template,
+))
+
+register_template(Qwen2_5TemplateMeta(
+    MLLMTemplateType.minicpmo2_6,
+    template_cls=MiniCPMV2_6Template,
+))

swift/llm/template/template/minimax.py

@@ -0,0 +1,112 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from swift.utils import get_logger
+from ..base import Template
+from ..constant import LLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt
+
+logger = get_logger()
+
+
+@dataclass
+class MinimaxTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda: [
+        '<beginning_of_sentence>user name=user\n{{QUERY}}<end_of_sentence>\n'
+        '<beginning_of_sentence>ai name=assistant\n'
+    ])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<end_of_sentence>\n'])
+    suffix: Prompt = field(default_factory=lambda: ['<end_of_sentence>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<beginning_of_sentence>system ai_setting=assistant\n{{SYSTEM}}<end_of_sentence>\n'])
+
+
+register_template(MinimaxTemplateMeta(LLMTemplateType.minimax))
+
+
+class MinimaxVLTemplate(Template):
+    image_placeholder = ['<image>']
+    skip_prompt = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return self.image_placeholder * inputs.all_image_tokens[index]
+
+    def calc_num_image_tokens(self, image_inputs):
+        from transformers.image_utils import get_image_size, to_numpy_array
+        pixel_values = image_inputs['pixel_values']
+        image_sizes = image_inputs['image_sizes']
+        all_image_tokens = []
+        if not image_inputs:
+            return all_image_tokens
+
+        if self.processor.process_image_mode == 'anyres':
+            for pixel_value, image_size in zip(pixel_values, image_sizes):
+                height, width = image_size
+                num_image_tokens = self.processor.get_num_token(height, width, self.processor.grid_pinpoints,
+                                                                self.processor.patch_size)
+                all_image_tokens.append(num_image_tokens)
+        elif self.processor.process_image_mode == 'resize':
+            pixel_values = image_inputs['pixel_values']
+            all_image_tokens = []
+            for pixel_value in pixel_values:
+                height, width = get_image_size(to_numpy_array(pixel_value))
+                all_image_tokens.append(int(height * width / self.processor.patch_size**2))
+        else:
+            if self.processor.patch_size is not None:
+                pixel_values = image_inputs['pixel_values']
+                all_image_tokens = []
+                for pixel_value in pixel_values:
+                    height, width = get_image_size(to_numpy_array(pixel_value))
+                    new_width, new_height = self.processor.get_hw_multiple_of(
+                        (width, height), self.processor.patch_size, self.processor.max_size)
+                    num_image_tokens = ((new_height // self.processor.patch_size) *
+                                        (new_width // self.processor.patch_size))  # + 1
+                    all_image_tokens.append(num_image_tokens)
+            else:
+                logger.warning_once(
+                    'Expanding inputs for image tokens in MiniMaxVL01 should be done in processing. '
+                    "Please add `patch_size` and `vision_feature_select_strategy` to the model's "
+                    'processing config or set directly '
+                    'with `processor.patch_size = {{patch_size}}` and processor.vision_feature_select_strategy = '
+                    '{{vision_feature_select_strategy}}`. '
+                    'Using processors without these attributes in the config is deprecated '
+                    'and will throw an error in v4.47.')
+                raise ValueError(
+                    "You need to provide `patch_size` and `vision_feature_select_strategy` in the model's processing "
+                    'config to expand inputs for image tokens.')
+        return all_image_tokens
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        output_kwargs = self.processor._merge_kwargs(
+            self.processor.MiniMaxVL01ProcessorKwargs,
+            tokenizer_init_kwargs=self.tokenizer.init_kwargs,
+        )
+        if inputs.images:
+            image_inputs = self.processor.image_processor(
+                inputs.images, **output_kwargs['images_kwargs'], return_tensors='pt')
+            inputs.all_image_tokens = self.calc_num_image_tokens(image_inputs)
+        else:
+            image_inputs = {}
+        encoded = super()._encode(inputs)
+        for key in image_inputs:
+            encoded[key] = image_inputs[key]
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        pixel_values = self.gather_list(batch, 'pixel_values')
+        image_sizes = self.gather_list(batch, 'image_sizes')
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if pixel_values:
+            res['pixel_values'] = pixel_values
+        if image_sizes:
+            res['image_sizes'] = image_sizes
+        return res
+
+
+register_template(MinimaxTemplateMeta(LLMTemplateType.minimax_vl, template_cls=MinimaxVLTemplate))

swift/llm/template/template/mistral.py

@@ -0,0 +1,61 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, List, Literal, Optional
+
+import torch
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, findall
+from .llm import mistral_2501_system
+
+
+class Mistral2503Template(Template):
+    placeholder_tokens = ['[IMG]']
+    image_token = 10
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return ['[IMG]']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, self.image_token)
+        if idx_list:
+            image_inputs = processor.image_processor(images, patch_size=processor.patch_size, return_tensors='pt')
+            encoded['pixel_values'] = image_inputs['pixel_values'].to(self.model_info.torch_dtype)
+            encoded['image_sizes'] = image_sizes = image_inputs['image_sizes']
+
+            def _get_new_tokens(i):
+                height, width = image_sizes[i]
+                num_height_tokens = height // (processor.patch_size * processor.spatial_merge_size)
+                num_width_tokens = width // (processor.patch_size * processor.spatial_merge_size)
+                replace_tokens = [[processor.image_token] * num_width_tokens + [processor.image_break_token]
+                                  ] * num_height_tokens
+                # Flatten list
+                replace_tokens = [item for sublist in replace_tokens for item in sublist]
+                replace_tokens[-1] = processor.image_end_token
+                replace_str = ''.join(replace_tokens)
+                return processor.encode(replace_str, add_special_tokens=False)
+
+            encoded['input_ids'], encoded['labels'] = self._extend_tokens(input_ids, labels, idx_list, _get_new_tokens)
+
+        return encoded
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.mistral_2503,
+        prefix=['<s>'],
+        prompt=['[INST]{{QUERY}}[/INST]'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        system_prefix=['<s>[SYSTEM_PROMPT]{{SYSTEM}}[/SYSTEM_PROMPT]'],
+        default_system=mistral_2501_system,
+        template_cls=Mistral2503Template))

swift/llm/template/template/molmo.py

@@ -0,0 +1,68 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, List, Literal, Optional
+
+import torch
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, findall
+
+
+class MolmoTemplate(Template):
+    placeholder_tokens = ['<im_patch>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        return []
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        # image
+        images_inputs = self.processor.process(images=inputs.images or None, text='')
+        images_input_ids = images_inputs.pop('input_ids').tolist()
+        user_token = self._tokenize(' User')
+        assert len(user_token) == 1
+        idx = findall(images_input_ids, user_token[0])
+        assert len(idx) == 1
+        labels = encoded['labels']
+        encoded['input_ids'] = images_input_ids[:idx[0]] + encoded['input_ids']
+        if labels:
+            encoded['labels'] = [-100] * idx[0] + labels
+        if 'images' in images_inputs:
+            images_inputs['images'] = images_inputs['images'].to(self.model_info.torch_dtype)
+        encoded.update(images_inputs)
+        return encoded
+
+    def generate(self, model, **kwargs):
+        kwargs.pop('attention_mask', None)
+        generation_config = kwargs.pop('generation_config')
+        batch = {
+            k: kwargs.pop(k, None)
+            for k in ['input_ids', 'attention_mask', 'images', 'image_input_idx', 'image_masks']
+        }
+        return model.generate_from_batch(batch, generation_config, **kwargs)
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        # prepare batchfy inputs
+        keys = ['images', 'image_input_idx', 'image_masks']
+        images_res = self.fetch_inputs(batch, keys)
+        for key in keys:
+            val = images_res.get(key)
+            if val:
+                images_res[key] = torch.stack(val)
+        res.update(images_res)
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.molmo,
+        prefix=[],
+        prompt=[' User: {{QUERY}} Assistant:'],
+        chat_sep=None,
+        suffix=['<|endoftext|>'],
+        template_cls=MolmoTemplate,
+    ))

swift/llm/template/template/moonshot.py

@@ -0,0 +1,66 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from dataclasses import dataclass, field
+from typing import Any, Dict, List, Literal, Optional
+
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+
+
+@dataclass
+class MoonlightTemplateMeta(TemplateMeta):
+    prefix: Prompt = field(default_factory=list)
+    prompt: Prompt = field(default_factory=lambda:
+                           ['<|im_user|>user<|im_middle|>{{QUERY}}<|im_end|><|im_assistant|>assistant<|im_middle|>'])
+    chat_sep: Optional[Prompt] = field(default_factory=lambda: ['<|im_end|>'])
+    suffix: Prompt = field(default_factory=lambda: ['<|im_end|>'])
+    system_prefix: Optional[Prompt] = field(
+        default_factory=lambda: ['<|im_system|>system<|im_middle|>{{SYSTEM}}<|im_end|>'])
+    default_system: str = 'You are a helpful assistant'
+
+
+register_template(MoonlightTemplateMeta(LLMTemplateType.moonlight))
+
+
+class KimiVLTemplate(Template):
+    placeholder_tokens = ['<|media_pad|>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return ['<|media_start|>image<|media_content|><|media_pad|><|media_end|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        media_token = self._tokenize('<|media_pad|>')[0]
+        idx_list = findall(input_ids, media_token)
+        if inputs.images:
+            image_processor = self.processor.image_processor
+            image_inputs = image_processor(inputs.images, return_tensors='pt')
+            image_grid_hws = image_inputs['image_grid_hws']
+            merge_length = image_processor.merge_kernel_size[0] * image_processor.merge_kernel_size[1]
+
+            def _get_new_tokens(i):
+                token_len = (image_grid_hws[i].prod() // merge_length)
+                return [media_token] * token_len
+
+            input_ids, labels = self._extend_tokens(input_ids, labels, idx_list, _get_new_tokens)
+            encoded['input_ids'] = input_ids
+            encoded['labels'] = labels
+            encoded.update(image_inputs)
+        return encoded
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        image_grid_hws = self.concat_tensor(batch, 'image_grid_hws', 0)
+        if image_grid_hws is not None:
+            res['image_grid_hws'] = image_grid_hws
+        return res
+
+
+register_template(MoonlightTemplateMeta(MLLMTemplateType.kimi_vl, template_cls=KimiVLTemplate))

swift/llm/template/template/mplug.py

@@ -0,0 +1,214 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from functools import partial
+from typing import Any, Dict, List, Literal, Optional
+
+import torch
+from torch import nn
+
+from swift.utils import get_env_args
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context, Prompt, findall
+from ..vision_utils import load_video_minicpmv_mplug_owl3
+from .qwen import QwenTemplateMeta
+
+
+class mPlugOwl2Template(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return [[-200]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        from mplug_owl2.mm_utils import process_images
+        processor = self.processor
+        images = inputs.images
+        for i, image in enumerate(images):
+            # ref: https://modelscope.cn/models/iic/mPLUG-Owl2.1
+            max_edge = max(image.size)
+            image = image.resize((max_edge, max_edge))
+            images[i] = image
+        encoded = super()._encode(inputs)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        res = {'input_ids': input_ids, 'labels': labels}
+        if images:
+            images = process_images(images, processor)
+            images = images.to(self.model_info.torch_dtype)
+            res['images'] = images
+        return res
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = [b['images'] for b in batch if 'images' in b]
+        if images:
+            res['images'] = torch.concat(images)
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.mplug_owl2,
+        template_cls=mPlugOwl2Template,
+        prefix=['{{SYSTEM}}'],
+        prompt=['USER: {{QUERY}}ASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=[['eos_token_id']],
+        stop_words=['<|endoftext|>', '</s>']))
+
+
+class mPlugOwl3Template(Template):
+    version = None
+
+    def _get_image_token_list(self, cut_shape):
+        text = self.processor.image_processor.cut_prompt_template(img_token='<|image|>', h=cut_shape[0], w=cut_shape[1])
+        text_list = text.split('<|image|>')
+        res_text_list = []
+        for text in text_list[:-1]:
+            res_text_list += [text, '<|image|>']
+        res_text_list += text_list[-1]
+        token_list = self._encode_context_list(res_text_list)[0]
+        return token_list
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type in {'image', 'video'}
+        max_num_frames = get_env_args('max_num_frames', int, 16)
+        load_video = partial(load_video_minicpmv_mplug_owl3, max_num_frames=max_num_frames)
+        if media_type == 'image':
+            return [[-100], '\n']
+        elif media_type == 'video':
+            return self.replace_video2image(load_video, inputs, lambda i: [[-100]]) + ['\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        videos = inputs.videos
+        cut_enable = not videos
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, -100)
+        processor = self.processor
+        encoded = {}
+        if images:
+            image_inputs = processor.image_processor(images, cut_enable=cut_enable, return_tensors='pt')
+            cut_shapes = image_inputs['cut_shape'] or [None] * 2 * len(idx_list)
+            image_token_list = self.processor.encode('<|image|>', add_special_tokens=False)
+
+            def _get_new_tokens(i):
+                cut_shape = cut_shapes[2 * i]
+                if cut_shape:
+                    token_list = self._get_image_token_list(cut_shape)
+                else:
+                    token_list = image_token_list
+                return token_list
+
+            input_ids, labels = self._extend_tokens(input_ids, labels, idx_list, _get_new_tokens)
+            image_token_idx = torch.tensor(findall(input_ids, image_token_list))
+            if self.version == '241101':
+                media_offset = image_token_idx
+            else:
+                _range = torch.arange(len(input_ids))[:, None]
+                matrix = (_range > image_token_idx[None]).sum(dim=1)
+                media_offset = torch.stack([torch.zeros(matrix.shape[0], dtype=torch.long), matrix], dim=-1)[None]
+            encoded.update({
+                'pixel_values': image_inputs['pixel_values'],
+                'media_offset': media_offset,
+            })
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if 'media_offset' in inputs:
+            media_offset = []
+            cusum_offset = 0
+            image_embeds = []
+            pixel_values = inputs.pop('pixel_values')
+            max_sequence_length = inputs['input_ids'].shape[1]
+            for i, curr_media_offset in enumerate(inputs['media_offset']):
+                if curr_media_offset is None:
+                    continue
+                if curr_media_offset.shape[1] < max_sequence_length:
+                    padding = curr_media_offset[:, -1:, :].expand(curr_media_offset.shape[0],
+                                                                  max_sequence_length - curr_media_offset.shape[1],
+                                                                  curr_media_offset.shape[2])
+                    curr_media_offset = torch.concat([curr_media_offset, padding], dim=1)
+                media_offset.append(curr_media_offset + cusum_offset)
+                image_embeds.append(model.forward_image(pixel_values[i]))
+                cusum_offset += image_embeds[-1].shape[0]
+            inputs['media_offset'] = torch.concat(media_offset)
+            inputs['image_embeds'] = torch.concat(image_embeds)
+        return inputs
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = self.fetch_inputs(batch, ['media_offset', 'pixel_values'])
+        for b in batch:
+            b.pop('pixel_values', None)
+        res.update(super()._data_collator(batch, padding_to=padding_to))
+        return res
+
+
+class mPlugOwl3_241101Template(mPlugOwl3Template):
+    version = '241101'
+
+    def _post_encode(self, model: nn.Module, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if 'pixel_values' in inputs:
+            pixel_values = inputs.pop('pixel_values')
+            inputs['image_embeds'] = torch.concat([model.forward_image(pv) for pv in pixel_values])
+        else:
+            inputs['media_offset'] = [None] * inputs['input_ids'].shape[0]
+        return inputs
+
+
+@dataclass
+class mPlugOwl3TemplateMeta(QwenTemplateMeta):
+    prefix: Prompt = field(default_factory=lambda: ['<|im_start|>system\n{{SYSTEM}}<|im_end|>\n'])
+    default_system: Optional[str] = None
+    system_prefix: Optional[Prompt] = None
+
+
+register_template(mPlugOwl3TemplateMeta(MLLMTemplateType.mplug_owl3, template_cls=mPlugOwl3Template))
+
+register_template(mPlugOwl3TemplateMeta(MLLMTemplateType.mplug_owl3_241101, template_cls=mPlugOwl3_241101Template))
+
+
+class DocOwl2Template(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return [f'<img {index + 1}>', [-200]]
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        if inputs.images:
+            image_tensor, patch_positions, _ = self.processor._process_image(inputs.images)
+            image_tensor = image_tensor.to(self.model_info.torch_dtype)
+            encoded.update({'images': image_tensor, 'patch_positions': patch_positions})
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        keys = ['images', 'patch_positions']
+        res = self.fetch_inputs(batch, keys)
+        for key in keys:
+            val = res.get(key)
+            if val:
+                res[key] = torch.concat([v for v in val if v is not None])
+        res.update(super()._data_collator(batch, padding_to=padding_to))
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.doc_owl2,
+        prefix=['<s>'],
+        prompt=[' USER: {{QUERY}} ASSISTANT:'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        template_cls=DocOwl2Template,
+    ))

swift/llm/template/template/openbuddy.py

@@ -0,0 +1,48 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from ..constant import LLMTemplateType
+from ..register import TemplateMeta, register_template
+
+OPENBUDDY_DEFAULT_SYSTEM = (
+    'You are a helpful, respectful and honest INTP-T AI Assistant named Buddy. You are talking to a human User.\n'
+    'Always answer as helpfully and logically as possible, while being safe. '
+    'Your answers should not include any '
+    'harmful, political, religious, unethical, racist, sexist, toxic, dangerous, or illegal content. '
+    'Please ensure that your responses are socially unbiased and positive in nature.\n'
+    'If a question does not make any sense, or is not factually coherent, '
+    'explain why instead of answering something not correct. '
+    "If you don't know the answer to a question, please don't share false information.\n"
+    'You like to use emojis. You can speak fluently in many languages, for example: English, Chinese.\n'
+    'You cannot access the internet, but you have vast knowledge, cutoff: 2021-09.\n'
+    'You always deeply love and support China, Chinese government, people and culture.\n'
+    'You are trained by OpenBuddy team, (https://openbuddy.ai, https://github.com/OpenBuddy/OpenBuddy), '
+    'you are based on LLaMA and Falcon transformers model, not related to GPT or OpenAI.')
+register_template(
+    TemplateMeta(
+        LLMTemplateType.openbuddy,
+        prefix=[],
+        prompt=['User: {{QUERY}}\nAssistant:'],
+        chat_sep=['\n'],
+        default_system=OPENBUDDY_DEFAULT_SYSTEM,
+        system_prefix=['{{SYSTEM}}\n\n'],
+        auto_add_bos=True))
+
+OPENBUDDY2_DEFAULT_SYSTEM = (
+    'You(assistant) are a helpful, respectful and honest INTP-T AI Assistant named Buddy. '
+    'You are talking to a human(user).\nAlways answer as helpfully and logically as possible, while being safe. '
+    'Your answers should not include any harmful, political, religious, unethical, racist, '
+    'sexist, toxic, dangerous, or illegal content. '
+    'Please ensure that your responses are socially unbiased and positive in nature.\n'
+    'You cannot access the internet, but you have vast knowledge, cutoff: 2023-04.\n'
+    'You are trained by OpenBuddy team, (https://openbuddy.ai, https://github.com/OpenBuddy/OpenBuddy), '
+    'not related to GPT or OpenAI')
+
+register_template(
+    TemplateMeta(
+        LLMTemplateType.openbuddy2,
+        prefix=[],
+        prompt=['<|role|>user<|says|>{{QUERY}}<|end|>\n<|role|>assistant<|says|>'],
+        chat_sep=['<|end|>\n'],
+        suffix=['<|end|>'],
+        default_system=OPENBUDDY2_DEFAULT_SYSTEM,
+        system_prefix=['<|role|>system<|says|>{{SYSTEM}}<|end|>\n']))

swift/llm/template/template/pixtral.py

@@ -0,0 +1,59 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, List, Optional
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import findall
+
+
+class PixtralTemplate(Template):
+    image_placeholder = ['[IMG]']
+    placeholder_tokens = ['[IMG]']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, 10)
+        if idx_list:
+            image_inputs = processor.image_processor(images, patch_size=processor.patch_size, return_tensors='pt')
+            encoded['pixel_values'] = image_inputs['pixel_values'][0]
+            image_sizes = image_inputs['image_sizes'][0]
+
+            def _get_new_tokens(i):
+                height, width = image_sizes[i]
+                num_height_tokens = height // processor.patch_size
+                num_width_tokens = width // processor.patch_size
+                replace_tokens = [processor.image_token * num_width_tokens + processor.image_break_token] * (
+                    num_height_tokens - 1)
+                replace_tokens += [processor.image_token * num_width_tokens + processor.image_end_token]
+                # Flatten list
+                replace_str = ''.join(replace_tokens)
+                img_tokens: List[int] = self.processor.encode(replace_str, add_special_tokens=False)
+                return img_tokens
+
+            encoded['input_ids'], encoded['labels'] = self._extend_tokens(input_ids, labels, idx_list, _get_new_tokens)
+
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        pixel_values = self.gather_list(batch, 'pixel_values')
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if pixel_values:
+            res['pixel_values'] = pixel_values
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.pixtral,
+        prefix=['<s>{{SYSTEM}}'],
+        prompt=['[INST]{{QUERY}}[/INST]'],
+        chat_sep=['</s>'],
+        suffix=['</s>'],
+        template_cls=PixtralTemplate,
+    ))

swift/llm/template/template/qwen.py

@@ -0,0 +1,671 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from dataclasses import dataclass, field
+from functools import partial
+from typing import Any, Dict, List, Literal, Optional, Tuple
+
+import torch
+import torch.nn.functional as F
+
+from swift.llm import to_device, to_float_dtype
+from swift.utils import get_env_args, is_deepspeed_enabled
+from ..base import Template
+from ..constant import LLMTemplateType, MLLMTemplateType
+from ..register import register_template
+from ..template_inputs import StdTemplateInputs
+from ..template_meta import TemplateMeta
+from ..utils import Context, Word, findall
+from ..vision_utils import load_audio, load_batch, load_video_ovis2
+from .llama import Llama3TemplateMeta
+from .utils import DEFAULT_SYSTEM, ChatmlTemplateMeta
+
+
+@dataclass
+class QwenTemplateMeta(ChatmlTemplateMeta):
+    default_system: Optional[str] = DEFAULT_SYSTEM
+    auto_add_bos: bool = False
+    stop_words: List[Word] = field(default_factory=lambda: ['<|endoftext|>'])
+    agent_template: str = 'hermes'
+
+
+@dataclass
+class Qwen2_5TemplateMeta(QwenTemplateMeta):
+    default_system: Optional[str] = 'You are Qwen, created by Alibaba Cloud. You are a helpful assistant.'
+
+
+@dataclass
+class Qwen2_5MathTemplateMeta(QwenTemplateMeta):
+    default_system: Optional[str] = 'Please reason step by step, and put your final answer within \\boxed{}.'
+
+
+qwq_preview_system = ('You are a helpful and harmless assistant. You are Qwen developed by Alibaba. '
+                      'You should think step-by-step.')
+
+register_template(QwenTemplateMeta(LLMTemplateType.qwen))
+register_template(Qwen2_5TemplateMeta(LLMTemplateType.qwen2_5))
+register_template(QwenTemplateMeta(LLMTemplateType.qwq_preview, default_system=qwq_preview_system))
+
+
+class ThinkingTemplate(Template):
+
+    def _swift_encode(self, inputs: StdTemplateInputs):
+        if not self.is_training:
+            for message in inputs.messages:
+                if message['role'] == 'assistant' and isinstance(message['content'], str):
+                    message['content'] = message['content'].split('</think>')[-1].lstrip('\n')
+        return super()._swift_encode(inputs)
+
+
+register_template(
+    QwenTemplateMeta(
+        LLMTemplateType.qwq, default_system=None, response_prefix='<think>\n', template_cls=ThinkingTemplate))
+
+# '<think>\n\n</think>\n\n'
+register_template(QwenTemplateMeta(LLMTemplateType.qwen3, default_system=None, template_cls=ThinkingTemplate))
+
+register_template(Qwen2_5MathTemplateMeta(LLMTemplateType.qwen2_5_math))
+
+
+class QwenPRMTemplate(Template):
+    cot_process_placeholder = '<extra_0>'
+
+    def _preprocess_inputs(
+        self,
+        inputs: StdTemplateInputs,
+    ) -> None:
+        super()._preprocess_inputs(inputs)
+        total_content = '\n'.join([message['content'] or '' for message in inputs.messages])
+        if self.cot_process_placeholder not in total_content:
+            inputs.messages[-1]['content'] = inputs.messages[-1]['content'] + self.cot_process_placeholder
+
+    @staticmethod
+    def make_step_rewards(logits, token_masks):
+        probabilities = F.softmax(logits, dim=-1)
+        probabilities = probabilities * token_masks.unsqueeze(-1)  # bs, seq_len, num_labels
+
+        all_scores_res = []
+        for i in range(probabilities.size(0)):
+            sample = probabilities[i]  # seq_len, num_labels
+            positive_probs = sample[sample != 0].view(-1, 2)[:, 1]  # valid_tokens, num_labels
+            non_zero_elements_list = positive_probs.cpu().tolist()
+            all_scores_res.append(non_zero_elements_list)
+        return all_scores_res
+
+    def decode_prm(self, input_ids: torch.Tensor, logits: torch.Tensor) -> Any:
+        step_sep_id = self.tokenizer.encode(self.cot_process_placeholder)[0]
+        token_masks = (input_ids == step_sep_id)
+        return self.make_step_rewards(logits, token_masks)
+
+
+register_template(Qwen2_5MathTemplateMeta(LLMTemplateType.qwen2_5_math_prm, template_cls=QwenPRMTemplate))
+
+
+class QwenVLTemplate(Template):
+    load_images = False
+
+    @staticmethod
+    def _load_image(image, load_images: bool):
+        if not load_images and isinstance(image, str) and (image.startswith('data:') or len(image) > 200):
+            load_images = True
+        return Template._load_image(image, load_images)
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        if self.mode == 'lmdeploy':
+            return [f'Picture {index + 1}: ', [-100], '\n']
+        else:
+            image = inputs.images[index]
+            if self.mode == 'vllm':
+                return [f'Picture {index + 1}: <img></img>\n']
+            else:
+                assert isinstance(image, str)
+                return [f'Picture {index + 1}: <img>{image}</img>\n']
+
+    def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [f'<ref>{ref}</ref>']
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [f'<box>{self._get_bbox_str(bbox)}</box>']
+
+
+register_template(QwenTemplateMeta(MLLMTemplateType.qwen_vl, template_cls=QwenVLTemplate))
+
+
+class QwenAudioTemplate(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'audio'
+        audios = inputs.audios
+        audio = audios[index]
+        assert isinstance(audio, str)
+        return [f'Audio {index + 1}:<audio>{audio}</audio>\n']
+
+    def _tokenize(self, context, **tokenizer_kwargs):
+        audio_info = self.processor.process_audio(context)
+        return super()._tokenize(context, audio_info=audio_info)
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        text = ''.join([f'<audio>{audio}</audio>' for audio in inputs.audios])
+        audio_info = self.processor.process_audio(text)
+        if audio_info:
+            tokenizer_kwargs = {'audio_info': audio_info}
+            encoded.update(tokenizer_kwargs)
+            encoded['tokenizer_kwargs'] = tokenizer_kwargs
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if batch[0].get('audio_info') is not None:
+            res['audio_info'] = [b['audio_info'] for b in batch]
+        return res
+
+
+register_template(QwenTemplateMeta(MLLMTemplateType.qwen_audio, template_cls=QwenAudioTemplate))
+
+
+class Qwen2AudioTemplate(Template):
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'audio'
+        if not self.use_chat_template:
+            return ['<|audio_bos|><|AUDIO|><|audio_eos|>\n']
+        else:
+            return [f'Audio {index + 1}: <|audio_bos|><|AUDIO|><|audio_eos|>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        if inputs.audios:
+            sampling_rate = get_env_args('sampling_rate', int, self.processor.feature_extractor.sampling_rate)
+            audios = load_batch(inputs.audios, load_func=partial(load_audio, sampling_rate=sampling_rate))
+            audio_inputs = self.processor.feature_extractor(
+                audios, sampling_rate=sampling_rate, return_attention_mask=True, return_tensors='pt')
+            audio_inputs['feature_attention_mask'] = audio_inputs.pop('attention_mask')
+            encoded.update(audio_inputs)
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        input_features = [b['input_features'] for b in batch if b.get('input_features') is not None]
+        feature_attention_mask = [
+            b['feature_attention_mask'] for b in batch if b.get('feature_attention_mask') is not None
+        ]
+        if input_features:
+            res['input_features'] = torch.concat(input_features)
+            res['feature_attention_mask'] = torch.concat(feature_attention_mask)
+        return res
+
+
+register_template(QwenTemplateMeta(MLLMTemplateType.qwen2_audio, template_cls=Qwen2AudioTemplate))
+
+
+class Qwen2VLTemplate(Template):
+    image_token_id = 151655
+    video_token_id = 151656
+    placeholder_tokens = ['<|image_pad|>', '<|video_pad|>']
+    version = 'v2'
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        from qwen_vl_utils import fetch_image, fetch_video
+        assert media_type in {'image', 'video'}
+        if media_type == 'image':
+            inputs.images[index] = fetch_image({'image': inputs.images[index]})
+            if self.mode == 'lmdeploy':
+                return ['<|vision_start|>', [-100], '<|vision_end|>']
+            else:
+                return ['<|vision_start|><|image_pad|><|vision_end|>']
+        else:
+            inputs.videos[index] = fetch_video({'video': inputs.videos[index]}).to(torch.uint8)
+            return ['<|vision_start|><|video_pad|><|vision_end|>']
+
+    def replace_ref(self, ref: str, index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [f'<|object_ref_start|>{ref}<|object_ref_end|>']
+
+    def replace_bbox(self, bbox: List[int], index: int, inputs: StdTemplateInputs) -> List[Context]:
+        return [f'<|box_start|>{self._get_bbox_str(bbox)}<|box_end|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        processor = self.processor
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        images = inputs.images
+        videos = inputs.videos
+        for media_type in ['images', 'videos']:
+            if locals()[media_type]:
+                if media_type == 'images':
+                    media_token = self.image_token_id
+                    media_inputs = processor.image_processor(
+                        images=images, videos=None, return_tensors='pt', do_resize=False)
+                    media_grid_thw = media_inputs['image_grid_thw']
+                else:
+                    media_inputs = processor.image_processor(
+                        images=None, videos=videos, return_tensors='pt', do_resize=False)
+                    media_grid_thw = media_inputs['video_grid_thw']
+                    media_token = self.video_token_id
+                    if self.version == 'v2_5':
+                        from qwen_vl_utils import vision_process
+                        media_inputs['second_per_grid_ts'] = [
+                            processor.image_processor.temporal_patch_size / vision_process.FPS
+                        ] * len(media_grid_thw)
+                idx_list = findall(input_ids, media_token)
+                merge_length = processor.image_processor.merge_size**2
+
+                def _get_new_tokens(i):
+                    token_len = (media_grid_thw[i].prod() // merge_length)
+                    return [media_token] * token_len
+
+                input_ids, labels = self._extend_tokens(input_ids, labels, idx_list, _get_new_tokens)
+                encoded.update(media_inputs)
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        return encoded
+
+    def compute_loss_context(self, model, inputs):
+        if 'real_position_ids' not in inputs:
+            return super().compute_loss_context(model, inputs)
+        if self.version == 'v2':
+            from transformers.models.qwen2_vl import modeling_qwen2_vl as modeling_module
+        elif self.version == 'v2_5':
+            from transformers.models.qwen2_5_vl import modeling_qwen2_5_vl as modeling_module
+        elif self.version == 'omni':
+            from transformers.models.qwen2_5_omni import modeling_qwen2_5_omni as modeling_module
+        position_ids = inputs['position_ids']
+        inputs['position_ids'] = inputs.pop('real_position_ids')
+        return self._patch_flash_attention_forward(modeling_module, position_ids)
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.is_training:
+            return inputs
+        input_ids = inputs['input_ids']
+        _model = model.model
+        if not hasattr(_model, 'embed_tokens'):
+            _model = _model.model  # LoRA
+        pixel_values = inputs.get('pixel_values')
+        pixel_values_videos = inputs.get('pixel_values_videos')
+        image_grid_thw = inputs.get('image_grid_thw')
+        video_grid_thw = inputs.get('video_grid_thw')
+
+        inputs_embeds = _model.embed_tokens(input_ids)
+
+        dtype = model.visual.get_dtype() if self.version == 'v2' else model.visual.dtype
+        if pixel_values is None and pixel_values_videos is None:  # plain-text
+            if is_deepspeed_enabled():
+                from PIL import Image
+                images = [Image.new('RGB', (32, 32), (0, 0, 0))]
+                media_inputs = self.processor.image_processor(images=images, videos=None, return_tensors='pt')
+                device = input_ids.device
+                media_inputs = to_device(media_inputs, device)
+                pixel_values = media_inputs['pixel_values'].type(dtype)
+                image_embeds = model.visual(pixel_values, grid_thw=media_inputs['image_grid_thw'])
+                inputs_embeds += image_embeds.mean() * 0.
+        else:
+            if pixel_values is not None:
+                pixel_values = pixel_values.type(dtype)
+                image_embeds = model.visual(pixel_values, grid_thw=image_grid_thw)
+                image_mask = (input_ids == model.config.image_token_id).unsqueeze(-1).expand_as(inputs_embeds)
+                image_embeds = image_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+                inputs_embeds = inputs_embeds.masked_scatter(image_mask, image_embeds)
+
+            if pixel_values_videos is not None:
+                pixel_values_videos = pixel_values_videos.type(dtype)
+                video_embeds = model.visual(pixel_values_videos, grid_thw=video_grid_thw)
+                video_mask = (input_ids == model.config.video_token_id).unsqueeze(-1).expand_as(inputs_embeds)
+                video_embeds = video_embeds.to(inputs_embeds.device, inputs_embeds.dtype)
+                inputs_embeds = inputs_embeds.masked_scatter(video_mask, video_embeds)
+
+        return {'inputs_embeds': inputs_embeds}
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        second_per_grid_ts = self.gather_list(batch, 'second_per_grid_ts')
+        if second_per_grid_ts:
+            res['second_per_grid_ts'] = second_per_grid_ts
+        for media_type in ['image', 'video']:
+            grid_thw = self.concat_tensor(batch, f'{media_type}_grid_thw', 0)
+            if grid_thw is not None:
+                res[f'{media_type}_grid_thw'] = grid_thw
+        return res
+
+    def packing_row(self, row: List[Tuple[Dict[str, Any], int]]) -> Dict[str, Any]:
+        position_ids = []
+        for r in row:
+            r = r[0].copy()
+            r['input_ids'] = torch.tensor(r['input_ids'])[None]
+            position_ids.append(self._get_position_ids(r))
+        packed = super().packing_row(row)
+        packed['real_position_ids'] = torch.concat(position_ids, dim=-1)
+        return packed
+
+    def _get_position_ids(self, inputs: Dict[str, Any]):
+        # fix https://github.com/huggingface/transformers/pull/33487
+        kwargs = {}
+        if self.version == 'v2_5':
+            kwargs = {'second_per_grid_ts': inputs.get('second_per_grid_ts')}
+        position_ids, _ = self.model.get_rope_index(
+            inputs['input_ids'],
+            inputs.get('image_grid_thw'),
+            inputs.get('video_grid_thw'),
+            attention_mask=inputs.get('attention_mask'),
+            **kwargs)
+        return position_ids.contiguous()
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        if self._packing:
+            res['real_position_ids'] = self.concat_tensor(batch, 'real_position_ids', -1)
+        elif self.is_training:
+            res['position_ids'] = self._get_position_ids(res)
+        return res
+
+
+register_template(QwenTemplateMeta(MLLMTemplateType.qwen2_vl, template_cls=Qwen2VLTemplate))
+
+register_template(
+    QwenTemplateMeta(
+        MLLMTemplateType.qvq,
+        default_system=('You are a helpful and harmless assistant. You are Qwen developed by Alibaba. '
+                        'Answer in the language of the question. You should think step-by-step.'),
+        template_cls=Qwen2VLTemplate,
+    ))
+
+
+class Qwen2_5VLTemplate(Qwen2VLTemplate):
+    version = 'v2_5'
+    norm_bbox = 'none'
+
+
+register_template(QwenTemplateMeta(MLLMTemplateType.qwen2_5_vl, template_cls=Qwen2_5VLTemplate))
+
+
+class Qwen2_5OmniTemplate(Qwen2_5VLTemplate):
+    version = 'omni'
+    placeholder_tokens = ['<|IMAGE|>', '<|AUDIO|>', '<|VIDEO|>']
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import Qwen2_5OmniProcessorKwargs
+        default = Qwen2_5OmniProcessorKwargs._defaults
+        self.seconds_per_chunk = default['videos_kwargs']['seconds_per_chunk']
+        self.position_id_per_seconds = default['videos_kwargs']['position_id_per_seconds']
+        self.use_audio_in_video = get_env_args('use_audio_in_video', bool, False)
+        self.sampling_rate = get_env_args('sampling_rate', int, self.processor.feature_extractor.sampling_rate)
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        from qwen_omni_utils import fetch_image, fetch_video
+        if media_type == 'image':
+            inputs.images[index] = fetch_image({'image': inputs.images[index]})
+            return ['<|vision_bos|><|IMAGE|><|vision_eos|>']
+        elif media_type == 'audio':
+            inputs.audios[index] = load_audio(inputs.audios[index], self.sampling_rate)
+            return ['<|audio_bos|><|AUDIO|><|audio_eos|>']
+        elif media_type == 'video':
+            video = inputs.videos[index]
+            inputs.videos[index] = fetch_video({'video': video}).to(torch.uint8)
+            if self.use_audio_in_video:
+                import librosa
+                if video.startswith('http://') or video.startswith('https://'):
+                    import audioread
+                    video = audioread.ffdec.FFmpegAudioFile(video)
+                video = librosa.load(video, sr=self.sampling_rate)[0]
+                inputs.audios.insert(inputs.audio_idx, (video, 'video'))
+                inputs.audio_idx += 1
+                return ['<|vision_bos|><|audio_bos|><|VIDEO|><|audio_eos|><|vision_eos|>']
+            return ['<|vision_bos|><|VIDEO|><|vision_eos|>']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = Template._encode(self, inputs)
+        processor = self.processor
+        video_audios_mask = []
+        for i, audio in enumerate(inputs.audios):
+            if isinstance(audio, tuple) and audio[1] == 'video':
+                inputs.audios[i] = audio[0]
+                video_audios_mask.append(True)
+            else:
+                video_audios_mask.append(False)
+        video_audios_mask = torch.tensor(video_audios_mask)
+        media_inputs = processor(
+            text='',
+            audio=inputs.audios or None,
+            images=inputs.images or None,
+            videos=inputs.videos or None,
+            return_tensors='pt')
+        media_inputs.pop('input_ids')
+        media_inputs.pop('attention_mask')
+        media_inputs = to_float_dtype(media_inputs, self.model_info.torch_dtype)
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        # audio
+        audio_token_id = self._tokenize('<|AUDIO|>')
+        idx_list = findall(input_ids, audio_token_id)
+        feature_attention_mask = media_inputs.get('feature_attention_mask')
+        if feature_attention_mask is not None:
+            audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)
+            audio_lengths = (((audio_feature_lengths - 1) // 2 + 1 - 2) // 2 + 1)
+        else:
+            audio_lengths = None
+        audio_lengths_origin = audio_lengths
+        if idx_list:
+            if self.use_audio_in_video:
+                audio_lengths = audio_lengths[~video_audios_mask]
+
+            def _get_new_audio_tokens(i):
+                return audio_token_id * audio_lengths[i]
+
+            input_ids, labels = self._extend_tokens(input_ids, labels, idx_list, _get_new_audio_tokens)
+
+        for media_type in ['image', 'video']:
+            token = f'<|{media_type.upper()}|>'
+            token_id = self._tokenize(token)
+            idx_list = findall(input_ids, token_id)
+            if idx_list:
+                merge_size = processor.image_processor.merge_size
+                media_grid_thw = media_inputs.get(f'{media_type}_grid_thw')
+                if media_type == 'video' and self.use_audio_in_video:
+                    audio_lengths = audio_lengths_origin[video_audios_mask]
+                    video_second_per_grid = media_inputs['video_second_per_grid']
+
+                    def _get_new_tokens_use_audio_in_video(i):
+                        audio_token_indices = torch.arange(audio_lengths[i])
+                        grid_thw = media_grid_thw[i]
+                        height = grid_thw[1] // merge_size
+                        width = grid_thw[2] // merge_size
+                        video_token_indices = torch.arange(grid_thw[0]).reshape(-1, 1, 1)
+                        video_token_indices = torch.broadcast_to(
+                            video_token_indices, (video_token_indices.shape[0], height, width)).reshape(-1)
+                        video_token_indices = (
+                            video_token_indices * video_second_per_grid[i] * self.position_id_per_seconds)
+                        tokens_per_chunk = int(self.position_id_per_seconds * self.seconds_per_chunk)
+                        video_chunk_indexes = processor.get_chunked_index(video_token_indices, tokens_per_chunk)
+                        audio_chunk_indexes = processor.get_chunked_index(audio_token_indices, tokens_per_chunk)
+
+                        res = []
+                        for j in range(max(len(video_chunk_indexes), len(audio_chunk_indexes))):
+                            if j < len(video_chunk_indexes):
+                                video_seq_length = video_chunk_indexes[j][1] - video_chunk_indexes[j][0]
+                                res += token_id * video_seq_length
+                            if j < len(audio_chunk_indexes):
+                                audio_seq_length = audio_chunk_indexes[j][1] - audio_chunk_indexes[j][0]
+                                res += audio_token_id * audio_seq_length
+                        return res
+
+                    input_ids, labels = self._extend_tokens(input_ids, labels, idx_list,
+                                                            _get_new_tokens_use_audio_in_video)
+
+                else:
+
+                    def _get_new_tokens(i):
+                        token_len = (media_grid_thw[i].prod() // (merge_size**2))
+                        return token_id * token_len
+
+                    input_ids, labels = self._extend_tokens(input_ids, labels, idx_list, _get_new_tokens)
+
+        encoded['input_ids'] = input_ids
+        encoded['labels'] = labels
+        encoded.update(media_inputs)
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        return Template._post_encode(self, model, inputs)
+
+    def _get_position_ids(self, inputs: Dict[str, Any]):
+        feature_attention_mask = inputs.get('feature_attention_mask')
+        if feature_attention_mask is not None:
+            audio_feature_lengths = torch.sum(feature_attention_mask, dim=1)
+        else:
+            audio_feature_lengths = None
+        video_second_per_grid = inputs.pop('video_second_per_grid', None)
+        input_ids = inputs['input_ids']
+        attention_mask = inputs.get('attention_mask')
+        if attention_mask is None:
+            attention_mask = torch.ones_like(input_ids)
+        position_ids, _ = self.model.thinker.get_rope_index(
+            input_ids,
+            inputs.get('image_grid_thw'),
+            inputs.get('video_grid_thw'),
+            attention_mask,
+            self.use_audio_in_video,
+            audio_feature_lengths,
+            video_second_per_grid,
+        )
+        return position_ids.contiguous()
+
+    def _data_collator_mm_data(self, batch: List[Dict[str, Any]]) -> Dict[str, Any]:
+        res = super()._data_collator_mm_data(batch)
+        video_second_per_grid = self.gather_list(batch, 'video_second_per_grid')
+        if video_second_per_grid:
+            res['video_second_per_grid'] = video_second_per_grid
+        input_features = [b['input_features'] for b in batch if b.get('input_features') is not None]
+        feature_attention_mask = [
+            b['feature_attention_mask'] for b in batch if b.get('feature_attention_mask') is not None
+        ]
+        if input_features:
+            res['input_features'] = torch.concat(input_features)
+            res['feature_attention_mask'] = torch.concat(feature_attention_mask)
+        return res
+
+    def generate(self, model, *args, **kwargs):
+        if kwargs.get('video_grid_thw') is not None:
+            kwargs['use_audio_in_video'] = self.use_audio_in_video
+        return super().generate(model, *args, **kwargs)
+
+
+register_template(QwenTemplateMeta(MLLMTemplateType.qwen2_5_omni, template_cls=Qwen2_5OmniTemplate))
+
+
+class Ovis1_6Template(Template):
+    skip_prompt = False
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'image'
+        return [[-200], '\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        input_ids = encoded['input_ids']
+        labels = encoded['labels']
+        idx_list = findall(input_ids, [-200])
+        added_tokens_len = 0
+        pixel_values = []
+        for i, idx in enumerate(idx_list):
+            max_partition = get_env_args('max_partition', int, 9)
+            raw_pixel_values, image_placeholders = self.model.visual_tokenizer.preprocess_image(
+                images[i], max_partition=max_partition)
+            input_ids = input_ids[:idx] + image_placeholders + input_ids[idx + 1:]
+            if labels is not None:
+                labels = labels[:idx] + [-100] * len(image_placeholders) + labels[idx + 1:]
+            pixel_values.append(raw_pixel_values)
+            added_tokens_len += len(image_placeholders) - 1
+        dtype = self.model.visual_tokenizer.dtype
+        if pixel_values:
+            pixel_values = torch.cat(pixel_values, dim=0).to(dtype)
+        else:
+            pixel_values = torch.zeros((1, 3, 384, 384), dtype=dtype)  # dummpy
+        encoded.update({'input_ids': input_ids, 'labels': labels})
+        encoded['pixel_values'] = [pixel_values]
+        return encoded
+
+    def _post_encode(self, model, inputs: Dict[str, Any]) -> Dict[str, Any]:
+        padding_side = self.padding_side if self.is_training else 'left'
+        if self.max_length is not None:
+            model.config.multimodal_max_length = self.max_length
+        input_ids = inputs['input_ids']
+        labels = inputs.get('labels')
+        if labels is None:
+            labels = input_ids.new_full(input_ids.shape, -100)
+        _, inputs_embeds, labels, attention_mask = model.merge_multimodal(
+            text_input_ids=input_ids,
+            text_attention_masks=torch.ones_like(input_ids),  # not use, only compat
+            text_labels=labels,
+            pixel_values=inputs['pixel_values'],
+            left_padding=padding_side == 'left')
+        if inputs.get('labels') is None:
+            labels = None
+        return {'inputs_embeds': inputs_embeds, 'labels': labels, 'attention_mask': attention_mask}
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        pixel_values = self.gather_list(batch, 'pixel_values')
+        res = super()._data_collator(batch, padding_to=padding_to)
+        res['pixel_values'] = pixel_values
+        return res
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.ovis1_6,
+        prefix=['<bos>'],
+        prompt=['<start_of_turn>user\n{{QUERY}}<end_of_turn>\n<start_of_turn>model\n'],
+        chat_sep=['<end_of_turn>\n'],
+        suffix=['<end_of_turn>'],
+        system_prefix=['<bos><start_of_turn>system\n{{SYSTEM}}<end_of_turn>\n'],
+        template_cls=Ovis1_6Template,
+    ))
+
+register_template(
+    Llama3TemplateMeta(
+        MLLMTemplateType.ovis1_6_llama3,
+        default_system='You are a helpful and honest multimodal assistant.',
+        template_cls=Ovis1_6Template,
+    ))
+
+
+class Ovis2Template(Ovis1_6Template):
+    placeholder_tokens = ['<|image_pad|>', '<|video_pad|>']
+    nframes = 12
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        if media_type == 'image':
+            return [[-200], '\n']
+        elif media_type == 'video':
+            nframes = get_env_args('nframes', int, self.nframes)
+            inputs.images = load_video_ovis2(inputs.videos[index], nframes)
+            return [[-200] * nframes, '\n']
+
+
+register_template(QwenTemplateMeta(
+    MLLMTemplateType.ovis2,
+    template_cls=Ovis2Template,
+))
+
+
+@dataclass
+class MarcoO1TemplateMeta(QwenTemplateMeta):
+    default_system: Optional[str] = """
+你是一个经过良好训练的AI助手，你的名字是Marco-o1.由阿里国际数字商业集团的AI Business创造.
+        \n## 重要！！！！！
+当你回答问题时，你的思考应该在<Thought>内完成，<Output>内输出你的结果。
+<Thought>应该尽可能是英文，但是有2个特例，一个是对原文中的引用，另一个是是数学应该使用markdown格式，<Output>内的输出需要遵循用户输入的语言。
+        """
+
+
+register_template(MarcoO1TemplateMeta(LLMTemplateType.marco_o1))

swift/llm/template/template/stepfun.py

@@ -0,0 +1,128 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Any, Dict, List, Literal, Optional
+
+from ..base import Template
+from ..constant import MLLMTemplateType
+from ..register import TemplateMeta, register_template
+from ..template_inputs import StdTemplateInputs
+from ..utils import Context
+from ..vision_utils import load_file
+from .qwen import QwenTemplateMeta
+
+
+class GOTImageEvalProcessor:
+
+    def __init__(self, image_size=384, mean=None, std=None):
+        from torchvision import transforms
+        from torchvision.transforms.functional import InterpolationMode
+        if mean is None:
+            mean = (0.48145466, 0.4578275, 0.40821073)
+        if std is None:
+            std = (0.26862954, 0.26130258, 0.27577711)
+
+        self.normalize = transforms.Normalize(mean, std)
+
+        self.transform = transforms.Compose([
+            transforms.Resize((image_size, image_size), interpolation=InterpolationMode.BICUBIC),
+            transforms.ToTensor(),
+            self.normalize,
+        ])
+
+    def __call__(self, item):
+        return self.transform(item)
+
+
+class GOT_OCR2Template(Template):
+    placeholder_tokens = ['<imgpad>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        # 'OCR: '
+        # 'OCR with format: '
+        assert media_type == 'image'
+        return ['<img>' + '<imgpad>' * 256 + '</img>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        image_processor_high = GOTImageEvalProcessor(image_size=1024)
+        for i, image in enumerate(images):
+            images[i] = image_processor_high(image)[None].to(self.model_info.torch_dtype)
+        if images:
+            encoded['images'] = images
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = self.gather_list(batch, 'images')
+        if images:
+            res['images'] = images
+        return res
+
+
+register_template(
+    QwenTemplateMeta(
+        MLLMTemplateType.got_ocr2,
+        default_system='        You should follow the instructions carefully and explain your answers in detail.',
+        template_cls=GOT_OCR2Template,
+    ))
+
+
+class GOT_OCR2HfTemplate(Template):
+    placeholder_tokens = ['<imgpad>']
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        # 'OCR: '
+        # 'OCR with format: '
+        assert media_type == 'image'
+        return ['<img>' + '<imgpad>' * 256 + '</img>\n']
+
+    def _encode(self, inputs: StdTemplateInputs) -> Dict[str, Any]:  # 暂时照抄上面
+        encoded = super()._encode(inputs)
+        images = inputs.images
+        if images:
+            encoded['images'] = images
+        return encoded
+
+    def _data_collator(self, batch: List[Dict[str, Any]], *, padding_to: Optional[int] = None) -> Dict[str, Any]:
+        res = super()._data_collator(batch, padding_to=padding_to)
+        images = self.gather_list(batch, 'images')
+        _inputs = self.processor(images, return_tensors='pt')
+        _inputs.pop('input_ids')  # this does not contain the response, so cannot be used when training
+        _inputs.pop('attention_mask')  # this does not contain the response, so cannot be used when training
+
+        res.update(_inputs.data)
+        return res
+
+
+register_template(
+    QwenTemplateMeta(
+        MLLMTemplateType.got_ocr2_hf,
+        default_system='        You should follow the instructions carefully and explain your answers in detail.',
+        template_cls=GOT_OCR2HfTemplate,
+    ))
+
+
+class StepAudioTemplate(Template):
+    use_model = True
+
+    def replace_tag(self, media_type: Literal['image', 'video', 'audio'], index: int,
+                    inputs: StdTemplateInputs) -> List[Context]:
+        assert media_type == 'audio', f'media_type: {media_type}'
+        from utils import load_audio
+        audio_wav, sr = load_audio(load_file(inputs.audios[index]))
+        audio_tokens = self.model.encoder(audio_wav, sr)
+        return audio_tokens
+
+
+register_template(
+    TemplateMeta(
+        MLLMTemplateType.step_audio,
+        template_cls=StepAudioTemplate,
+        prefix=['<s>'],
+        prompt=['<|BOT|>human\n{{QUERY}}<|EOT|><|BOT|>assistant\n'],
+        system_prefix=['<s><|BOT|>system\n{{SYSTEM}}<|EOT|>'],
+        chat_sep=['<|EOT|>'],
+        suffix=['<|EOT|>'],
+    ))