Add files using upload-large-folder tool

ms-swift/swift/tuners/adapter.py

@@ -0,0 +1,189 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import inspect
+import re
+import types
+from dataclasses import dataclass, field
+from typing import List, Union
+
+import torch
+from torch import nn
+from transformers.activations import ACT2CLS
+
+from swift.utils.torch_utils import find_sub_module, get_logger
+from .utils import ActivationMixin, SwiftAdapter, SwiftConfig, SwiftOutput
+
+logger = get_logger()
+
+
+@dataclass
+class AdapterConfig(SwiftConfig):
+    """
+    The configuration class for the adapter module.
+
+    Adapters project input tokens by an MLP layer.
+    'Parameter-Efficient Transfer Learning for NLP' by Houlsby et al.(2019)
+    See http://arxiv.org/abs/1902.00751
+
+    Args:
+        dim(`int`): The dimension of the hidden states
+        target_modules(`Union[str, List[str]]`): The feedforward module to be replaced.
+            in regex format if this argument is str, else will match with `end with` if List[str].
+        hidden_pos(`Union[str, int]`): The position of the hidden state to be passed into the adapter,
+            can be int (args) or str (kwargs)
+        method_name(`str`): The method to be replaced, default is `forward`
+        adapter_length: The length of the adapter length (intermediate length)
+        act_layer: The activation layer of the adapter
+    """
+
+    dim: int = field(default=None, metadata={'help': 'The dimension of the hidden states'})
+
+    target_modules: Union[str, List[str]] = field(
+        default=None,
+        metadata={
+            'help':
+            'The feedforward module to be replaced. in regex format if this argument is str, '
+            'else will match with `end with` if List[str].'
+        })
+
+    hidden_pos: Union[str, int] = field(
+        default=None,
+        metadata={
+            'help': 'The position of the hidden state to be passed into the adapter, can be int (args) or str (kwargs)'
+        })
+
+    method_name: str = field(default='forward', metadata={'help': 'The method to be replaced, default is `forward`'})
+
+    adapter_length: int = field(
+        default=128, metadata={'help': 'The length of the adapter length (intermediate length)'})
+
+    act_layer: str = field(default='gelu', metadata={'help': 'The activation layer of the adapter'})
+
+    def __post_init__(self):
+        from .mapping import SwiftTuners
+        self.swift_type = SwiftTuners.ADAPTER
+
+
+class Adapter(SwiftAdapter):
+
+    @staticmethod
+    def prepare_model(model: nn.Module, config: AdapterConfig, adapter_name: str) -> SwiftOutput:
+        """Prepare a model with `AdapterConfig`"""
+        module_keys = [key for key, _ in model.named_modules()]
+
+        for module_key in module_keys:
+            if isinstance(config.target_modules, str):
+                target_module_found = re.fullmatch(config.target_modules, module_key)
+            else:
+                target_module_found = any(module_key.endswith(target_key) for target_key in config.target_modules)
+
+            if target_module_found:  # noqa
+                module = model.get_submodule(module_key)
+
+                def _forward(self, *args, **kwargs):
+                    args = getattr(self, f'forward_origin_{adapter_name}')(*args, **kwargs)
+                    if isinstance(args, (tuple, list, dict)):
+                        if isinstance(config.hidden_pos, int):
+                            _type = type(args)
+                            args = list(args)
+                            args[config.hidden_pos] = getattr(self, f'adapter_{adapter_name}')(args[config.hidden_pos])
+                            args = _type(args)
+                        else:
+                            args[config.hidden_pos] = getattr(self, f'adapter_{adapter_name}')(args[config.hidden_pos])
+                    elif isinstance(args, torch.Tensor):
+                        args = getattr(self, f'adapter_{adapter_name}')(args)
+                    return args
+
+                def _feed_forward_chunk(self, attention_output):
+                    return _forward(self, attention_output)
+
+                # TODO The `config.method_name` method should not be replaced twice.
+
+                setattr(module, f'forward_origin_{adapter_name}', getattr(module, config.method_name))
+                num_args_in_forward_chunk_fn = len(
+                    inspect.signature(getattr(module, f'forward_origin_{adapter_name}')).parameters)
+                if config.method_name == 'feed_forward_chunk' and num_args_in_forward_chunk_fn == 1:
+                    setattr(module, config.method_name, types.MethodType(_feed_forward_chunk, module))
+                else:
+                    setattr(module, config.method_name, types.MethodType(_forward, module))
+                adapter_module = AdapterModule(config.dim, adapter_name, module_key, config.adapter_length,
+                                               ACT2CLS[config.act_layer])
+                setattr(module, f'adapter_{adapter_name}', adapter_module)
+                logger.info(f'Adapter modules(module_key): {module_key}.adapter_{adapter_name}')
+
+        def state_dict_callback(state_dict, adapter_name: str, **kwargs):
+            return {key: value for key, value in state_dict.items() if f'adapter_{adapter_name}' in key}
+
+        def mark_trainable_callback(model):
+            return
+
+        return SwiftOutput(
+            config=config, state_dict_callback=state_dict_callback, mark_trainable_callback=mark_trainable_callback)
+
+    @staticmethod
+    def activate_adapter(module: torch.nn.Module, adapter_name: str, activate: bool, offload: str = None):
+        modules = find_sub_module(module, f'adapter_{adapter_name}')
+        for _module in modules:
+            _module: ActivationMixin
+            _module: nn.Module
+            _module.set_activation(adapter_name, activate)
+            SwiftAdapter.save_memory(_module, adapter_name, _module.module_key, activate, offload)
+
+
+class AdapterModule(nn.Module, ActivationMixin):
+    """The implementation of adapter tuning method.
+
+    Adapters project input tokens by an MLP layer.
+    'Parameter-Efficient Transfer Learning for NLP' by Houlsby et al.(2019)
+    See http://arxiv.org/abs/1902.00751
+
+    Args:
+        dim: An integer indicating the embedding dimension.
+        adapter_length: An integer indicating the length of adapter tuning.
+    """
+
+    def __init__(
+        self,
+        dim,
+        adapter_name,
+        module_key,
+        adapter_length=None,
+        act_layer=nn.GELU,
+    ):
+        super(AdapterModule, self).__init__()
+        super(nn.Module, self).__init__(module_key)
+        self.dim = dim
+        self.adapter_name = adapter_name
+        self.adapter_length = adapter_length
+        self.linear1 = nn.Linear(dim, adapter_length)
+        self.act = act_layer()
+        self.linear2 = nn.Linear(adapter_length, dim)
+        self.init_weights()
+        self._prepared = False
+        self.mark_all_sub_modules_as_plugin()
+
+    def init_weights(self):
+
+        def _init_weights(m):
+            if isinstance(m, nn.Linear):
+                nn.init.xavier_uniform_(m.weight)
+                nn.init.normal_(m.bias, std=1e-6)
+
+        self.apply(_init_weights)
+
+    def forward(self, x, identity=None):
+        if not self.is_activated(self.adapter_name):
+            return x
+        if not self._prepared:
+            self.linear1.to(x.device)
+            self.act.to(x.device)
+            self.linear2.to(x.device)
+            self._prepared = True
+
+        x_dtype = x.dtype
+        x = x.to(self.linear1.weight.dtype)
+        out = self.linear2(self.act(self.linear1(x)))
+        if identity is None:
+            identity = x
+        identity = identity.to(out.dtype)
+        out = identity + out
+        return out.to(x_dtype)

ms-swift/swift/tuners/peft.py

@@ -0,0 +1,392 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Copyright 2023-present the HuggingFace Inc. team.
+import os.path
+from dataclasses import asdict, dataclass, field
+from functools import partial, reduce
+from types import MethodType
+from typing import Dict, Optional
+
+import json
+import peft
+import torch
+import torch.nn
+import transformers
+from modelscope import snapshot_download
+from peft import (AdaLoraConfig, BOFTConfig, BOFTModel, LoftQConfig, LoHaConfig, LoKrConfig, LoraModel, OFTConfig,
+                  PeftConfig, PeftModel, PeftModelForCausalLM, PeftModelForSeq2SeqLM,
+                  PeftModelForSequenceClassification, PeftModelForTokenClassification, PrefixTuningConfig,
+                  PromptEncoderConfig, PromptLearningConfig, PromptTuningConfig, VeraConfig, VeraModel, get_peft_config,
+                  get_peft_model, get_peft_model_state_dict)
+from peft.config import PeftConfigMixin
+from peft.tuners import lora
+from peft.tuners.adalora import AdaLoraModel, RankAllocator
+from peft.tuners.lora import Embedding
+from transformers import Trainer
+
+from swift.utils import get_logger
+
+try:
+    from peft import FourierFTModel
+except ImportError:
+    FourierFTModel = None
+
+try:
+    from peft import BoneModel
+except ImportError:
+    BoneModel = None
+
+logger = get_logger()
+dispatchers = []
+
+
+@dataclass
+class LoraConfig(peft.LoraConfig):
+    lora_dtype: Optional[str] = field(
+        default=None, metadata={'help': 'The lora dtype, default None means following the original layer\'s dtype'})
+
+    lorap_lr_ratio: Optional[float] = field(default=None, metadata={'help': 'The lr ratio of lora_B in lora+'})
+
+    lorap_emb_lr: float = field(default=1e-6, metadata={'help': 'The lr for embedding in lora+'})
+
+    def to_peft_config(self) -> peft.LoraConfig:
+        _dict = asdict(self)
+        _dict.pop('lora_dtype')
+        _dict.pop('lorap_lr_ratio')
+        _dict.pop('lorap_emb_lr')
+        return peft.LoraConfig(**_dict)
+
+    def save_pretrained(self, save_directory: str, **kwargs) -> None:
+        self.to_peft_config().save_pretrained(save_directory, **kwargs)
+        additional_args = {
+            'lora_dtype': self.lora_dtype,
+            'lorap_lr_ratio': self.lorap_lr_ratio,
+            'lorap_emb_lr': self.lorap_emb_lr,
+        }
+        with open(os.path.join(save_directory, 'additional_config.json'), 'w', encoding='utf-8') as f:
+            json.dump(additional_args, f)
+
+    @classmethod
+    def from_pretrained(cls, pretrained_model_name_or_path: str, subfolder: Optional[str] = None, **kwargs):
+        if hasattr(PeftConfigMixin, 'from_pretrained_origin'):
+            self = PeftConfigMixin.from_pretrained_origin(pretrained_model_name_or_path, subfolder, **kwargs)
+        else:
+            self = super(LoraConfig, cls).from_pretrained(pretrained_model_name_or_path, subfolder, **kwargs)
+
+        if type(self) == peft.LoraConfig:
+            self = LoraConfig(**self.to_dict())
+
+        if os.path.isfile(os.path.join(pretrained_model_name_or_path, 'additional_config.json')):
+            with open(
+                    os.path.join(pretrained_model_name_or_path, 'additional_config.json'), 'r', encoding='utf-8') as f:
+                _json = json.load(f)
+                for key, value in _json.items():
+                    setattr(self, key, value)
+
+        return self
+
+
+def _create_and_replace_hook(self, peft_config, adapter_name, target, *args, **kwargs):
+    all_supported_names = ('linear', )
+    all_supported_types = (torch.nn.Embedding, torch.nn.Conv2d, transformers.pytorch_utils.Conv1D, lora.Linear)
+    target_modules = getattr(peft_config, 'target_modules', None)
+    if target is None:
+        return
+
+    if isinstance(target_modules, str) and not any(
+        [name in target.__class__.__name__.lower()
+         for name in all_supported_names]) and not any([isinstance(target, type_) for type_ in all_supported_types]):
+        return
+
+    if target.__class__.__name__ == 'NonDynamicallyQuantizableLinear':
+        return
+
+    return self._create_and_replace_origin(peft_config, adapter_name, target, *args, **kwargs)
+
+
+def _convert_dtype(target: torch.nn.Module, adapter_name: str, lora_dtype: str):
+    if lora_dtype is not None:
+        torch_dtype = eval(f'torch.{lora_dtype}')
+        if hasattr(target, 'lora_A') and adapter_name in target.lora_A:
+            target.lora_A[adapter_name].to(torch_dtype)
+            target.lora_B[adapter_name].to(torch_dtype)
+        if hasattr(target, 'lora_embedding_A') and adapter_name in target.lora_embedding_A:
+            target.lora_embedding_A[adapter_name].to(torch_dtype)
+            target.lora_embedding_B[adapter_name].to(torch_dtype)
+
+
+def create_optimizer_param_groups(self: PeftModel, **defaults):
+    if not isinstance(self.peft_config[self.active_adapter],
+                      LoraConfig) or self.peft_config[self.active_adapter].lorap_lr_ratio is None:
+        return None
+
+    def get_module(name):
+        parent_idx = 2 if 'lora' in name else 1
+        module_names = name.split(sep='.')[:-parent_idx]
+        module = reduce(getattr, module_names, self.base_model)
+        return module
+
+    param_groups = {
+        'groupA': {},
+        'groupB': {},
+        'groupB_no_decay': {},
+        'embedding': {},
+    }
+
+    decay_parameters = Trainer.get_decay_parameter_names(None, self.base_model)
+    for name, param in self.base_model.named_parameters():
+        if not param.requires_grad:
+            continue
+
+        module = get_module(name)
+        if isinstance(module, Embedding):
+            param_groups['embedding'][name] = param
+        elif 'lora_B' in name or param.ndim == 1:
+            if name in decay_parameters:
+                param_groups['groupB'][name] = param
+            else:
+                param_groups['groupB_no_decay'][name] = param
+        else:
+            param_groups['groupA'][name] = param
+
+    lr = defaults['lr']
+    weight_decay = defaults.get('weight_decay', 0.0)
+
+    param_groups = [
+        {
+            'params': list(param_groups['groupA'].values()),
+            'weight_decay': weight_decay,
+            'lr': lr,
+        },
+        {
+            'params': list(param_groups['embedding'].values()),
+            'weight_decay': weight_decay,
+            'lr': self.peft_config[self.active_adapter].lorap_emb_lr,
+        },
+        {
+            'params': list(param_groups['groupB'].values()),
+            'weight_decay': weight_decay,
+            'lr': lr * self.peft_config[self.active_adapter].lorap_lr_ratio,
+        },
+        {
+            'params': list(param_groups['groupB_no_decay'].values()),
+            'weight_decay': 0.0,
+            'lr': lr * self.peft_config[self.active_adapter].lorap_lr_ratio,
+        },
+    ]
+    return param_groups
+
+
+def adalora_forward(self, *args, **kwargs):
+    from peft.utils.integrations import gather_params_ctx
+    outputs = self.model.forward(*args, **kwargs)
+
+    if (getattr(outputs, 'loss', None) is not None) and isinstance(outputs.loss, torch.Tensor):
+        # Calculate the orthogonal regularization
+        orth_reg_weight = self.peft_config[self.trainable_adapter_name].orth_reg_weight
+
+        if orth_reg_weight <= 0:
+            raise ValueError('orth_reg_weight should be greater than 0. ')
+
+        regu_loss = 0
+        num_param = 0
+        for n, p in self.model.named_parameters():
+            if ('lora_A' in n or 'lora_B' in n) and self.trainable_adapter_name in n:
+                if p.shape == torch.Size([0]):
+                    with gather_params_ctx(p, fwd_module=self):
+                        para_cov = p @ p.T if 'lora_A' in n else p.T @ p
+                else:
+                    para_cov = p @ p.T if 'lora_A' in n else p.T @ p
+                I = torch.eye(*para_cov.size(), out=torch.empty_like(para_cov))  # noqa: E741
+                I.requires_grad = False
+                num_param += 1
+                if isinstance(regu_loss, torch.Tensor):
+                    regu_loss = regu_loss.to(para_cov.device)
+                regu_loss += torch.norm(para_cov - I, p='fro')
+        if num_param > 0:
+            regu_loss = regu_loss / num_param
+        else:
+            regu_loss = 0
+        if isinstance(regu_loss, torch.Tensor) and isinstance(outputs.loss, torch.Tensor):
+            regu_loss = regu_loss.to(outputs.loss.device)
+        outputs.loss += orth_reg_weight * regu_loss
+    return outputs
+
+
+def adalora_mask_to_budget(self, model, budget):
+    value_ipt = {}
+    vector_ipt = {}
+    triplet_ipt = {}
+    # Get the importance score for A, E, B
+    for n, p in model.named_parameters():
+        if f'lora_A.{self.adapter_name}' in n:
+            entry_ipt = self._element_score(n)
+            comb_ipt = torch.mean(entry_ipt, dim=1, keepdim=True)
+            name_m = n.replace('lora_A', '%s')
+            if name_m not in vector_ipt:
+                vector_ipt[name_m] = [comb_ipt]
+            else:
+                vector_ipt[name_m].append(comb_ipt)
+        if f'lora_B.{self.adapter_name}' in n:
+            entry_ipt = self._element_score(n)
+            comb_ipt = torch.mean(entry_ipt, dim=0, keepdim=False).view(-1, 1)
+            name_m = n.replace('lora_B', '%s')
+            if name_m not in vector_ipt:
+                vector_ipt[name_m] = [comb_ipt]
+            else:
+                vector_ipt[name_m].append(comb_ipt)
+        if f'lora_E.{self.adapter_name}' in n:
+            entry_ipt = self._element_score(n)
+            name_m = n.replace('lora_E', '%s')
+            value_ipt[name_m] = entry_ipt
+
+    all_score = []
+    # Calculate the score for each triplet
+    for name_m in vector_ipt:
+        ipt_E = value_ipt[name_m]
+        ipt_AB = torch.cat(vector_ipt[name_m], dim=1)
+        sum_ipt = self._combine_ipt(ipt_E, ipt_AB)
+        name_E = name_m % 'lora_E'
+        triplet_ipt[name_E] = sum_ipt.view(-1, 1)
+        sum_ipt = sum_ipt.view(-1)
+        if all_score:
+            sum_ipt = sum_ipt.to(all_score[0].device)
+        all_score.append(sum_ipt)
+
+    # Get the threshold by ranking ipt
+    mask_threshold = torch.kthvalue(
+        torch.cat(all_score),
+        k=self.init_bgt - budget,
+    )[0].item()
+
+    rank_pattern = {}
+    # Mask the unimportant triplets
+    with torch.no_grad():
+        for n, p in model.named_parameters():
+            if f'lora_E.{self.adapter_name}' in n:
+                p.masked_fill_(triplet_ipt[n] <= mask_threshold, 0.0)
+                rank_pattern[n] = (~(triplet_ipt[n] <= mask_threshold)).view(-1).tolist()
+    return rank_pattern
+
+
+def keep_device_forward(self, *args, **kwargs):
+    x = args[0]
+    if self.weight.device != x.device:
+        return self.forward_origin(x.to(self.weight.device), *args[1:], **kwargs)
+    else:
+        return self.forward_origin(*args, **kwargs)
+
+
+def hot_patch_peft_module():
+    from peft.tuners.lora import LoraLayer
+    if hasattr('LoraModel', '_create_and_replace_origin'):
+        return
+
+    # Fix Lora does not support NonDynamicallyQuantizableLinear
+    LoraModel._create_and_replace_origin = LoraModel._create_and_replace
+    LoraModel._create_and_replace = _create_and_replace_hook
+    AdaLoraModel._create_and_replace_origin = AdaLoraModel._create_and_replace
+    AdaLoraModel._create_and_replace = _create_and_replace_hook
+    VeraModel._create_and_replace_origin = VeraModel._create_and_replace
+    VeraModel._create_and_replace = _create_and_replace_hook
+    BOFTModel._create_and_replace_origin = BOFTModel._create_and_replace
+    BOFTModel._create_and_replace = _create_and_replace_hook
+    if FourierFTModel is not None:
+        FourierFTModel._create_and_replace_origin = FourierFTModel._create_and_replace
+        FourierFTModel._create_and_replace = _create_and_replace_hook
+    if BoneModel is not None:
+        BoneModel._create_and_replace_origin = BoneModel._create_and_replace
+        BoneModel._create_and_replace = _create_and_replace_hook
+
+    # Support type conversion
+    def __new_init__(self, model: torch.nn.Module, config: Dict[str, LoraConfig], adapter_name: str):
+
+        self.__init_origin__(model, config, adapter_name)
+        active_adapters = self.active_adapter
+        if isinstance(active_adapters, str):
+            active_adapters = [active_adapters]
+        for active_adapter in active_adapters:
+            active_config = config[active_adapter] if isinstance(config, dict) else config
+            if hasattr(active_config, 'lora_dtype'):
+                for name, module in model.named_modules():
+                    if isinstance(module, LoraLayer):
+                        _convert_dtype(module, active_adapter, active_config.lora_dtype)
+                        for lora in list(module.lora_A.values()) + list(module.lora_B.values()):
+                            if not hasattr(lora, 'forward_origin'):
+                                lora.forward_origin = lora.forward
+                                lora.forward = MethodType(keep_device_forward, lora)
+
+    LoraModel.__init_origin__ = LoraModel.__init__
+    LoraModel.__init__ = __new_init__
+
+    # Support LoRA+
+    PeftModel.create_optimizer_param_groups = create_optimizer_param_groups
+
+    PeftConfigMixin.from_pretrained_origin = PeftConfigMixin.from_pretrained
+    PeftConfigMixin.from_pretrained = LoraConfig.from_pretrained
+
+    # Compatible with SwiftModel
+    def dummy_function(*args, **kwargs):
+        logger.warn(f'The function {kwargs["func"]} has no effects, consider using other functions.')
+
+    PeftModel.activate_adapter = PeftModel.set_adapter
+    PeftModel.deactivate_adapter = partial(dummy_function, func='deactivate_adapter')
+    PeftModel.set_active_adapters = partial(dummy_function, func='set_active_adapters')
+
+    # Fix adalora does not support device_map
+    AdaLoraModel.forward = adalora_forward
+    RankAllocator.mask_to_budget = adalora_mask_to_budget
+
+
+def get_wrapped_class(module_class):
+    """Get a custom wrapper class for peft classes to download the models from the ModelScope hub
+
+    Args:
+        module_class: The actual module class
+
+    Returns:
+        The wrapper
+    """
+
+    class PeftWrapper(module_class):
+
+        @classmethod
+        def from_pretrained(cls, model, model_id, *args, revision: Optional[str] = None, **kwargs):
+            if not os.path.exists(model_id):
+                model_id = snapshot_download(model_id, revision=revision)
+            return module_class.from_pretrained(model, model_id, *args, **kwargs)
+
+    PeftWrapper.__name__ = module_class.__name__
+    PeftWrapper.__qualname__ = module_class.__qualname__
+    return PeftWrapper
+
+
+def wrap_module(module):
+    if not hasattr(module, 'from_pretrained'):
+        return module
+
+    return get_wrapped_class(module)
+
+
+hot_patch_peft_module()
+PeftModel = wrap_module(PeftModel)
+PeftConfig = wrap_module(PeftConfig)
+PeftModelForSeq2SeqLM = wrap_module(PeftModelForSeq2SeqLM)
+PeftModelForSequenceClassification = wrap_module(PeftModelForSequenceClassification)
+PeftModelForTokenClassification = wrap_module(PeftModelForTokenClassification)
+PeftModelForCausalLM = wrap_module(PeftModelForCausalLM)
+PromptEncoderConfig = wrap_module(PromptEncoderConfig)
+PromptTuningConfig = wrap_module(PromptTuningConfig)
+PrefixTuningConfig = wrap_module(PrefixTuningConfig)
+PromptLearningConfig = wrap_module(PromptLearningConfig)
+LoraConfig = wrap_module(LoraConfig)
+AdaLoraConfig = wrap_module(AdaLoraConfig)
+LoHaConfig = wrap_module(LoHaConfig)
+LoKrConfig = wrap_module(LoKrConfig)
+LoftQConfig = wrap_module(LoftQConfig)
+OFTConfig = wrap_module(OFTConfig)
+BOFTConfig = wrap_module(BOFTConfig)
+VeraConfig = wrap_module(VeraConfig)
+OFTConfig = wrap_module(OFTConfig)
+get_peft_config = get_peft_config
+get_peft_model_state_dict = get_peft_model_state_dict
+get_peft_model = get_peft_model

ms-swift/swift/tuners/scetuning/scetuning_components.py

@@ -0,0 +1,127 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import math
+
+import torch
+import torch.nn as nn
+
+from swift.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def detach_tensors(feats):
+    if type(feats) in [list, tuple]:
+        feats = [detach_tensors(feat) if feat is not None else None for feat in feats]
+    elif isinstance(feats, dict):
+        feats = {key: detach_tensors(val) for key, val in feats.items()}
+    elif isinstance(feats, torch.Tensor):
+        feats = feats.detach()
+    else:
+        feats = feats.detach()
+    return feats
+
+
+def probe_tensors(module, feats, name):
+    feats = detach_tensors(feats)
+    setattr(module, name, feats)
+
+
+def probe_input_pre_hook(self, args):
+    input = args[0]
+    probe_tensors(self, input, 'probe_input_data')
+    return args
+
+
+def probe_output_hook(self, args, result):
+    output = result
+    probe_tensors(self, output, 'probe_output_data')
+    return output
+
+
+def choose_weight_type(weight_type, dim):
+    if weight_type == 'gate':
+        scaling = nn.Linear(dim, 1)
+    elif weight_type == 'scale':
+        scaling = nn.Parameter(torch.Tensor(1))
+        scaling.data.fill_(1)
+    elif weight_type == 'scale_channel':
+        scaling = nn.Parameter(torch.Tensor(dim))
+        scaling.data.fill_(1)
+    elif weight_type and weight_type.startswith('scalar'):
+        scaling = float(weight_type.split('_')[-1])
+    else:
+        scaling = None
+    return scaling
+
+
+def get_weight_value(weight_type, scaling, x):
+    if weight_type in ['gate']:
+        scaling = torch.mean(torch.sigmoid(scaling(x)), dim=1).view(-1, 1, 1)
+    elif weight_type in ['scale', 'scale_channel'] or weight_type.startswith('scalar'):
+        scaling = scaling
+    else:
+        scaling = None
+    return scaling
+
+
+class SCEAdapter(nn.Module):
+
+    def __init__(self,
+                 dim,
+                 adapter_length,
+                 adapter_type=None,
+                 adapter_weight=None,
+                 act_layer=nn.GELU,
+                 zero_init_last=True,
+                 use_bias=True):
+        super(SCEAdapter, self).__init__()
+        self.dim = dim
+        self.adapter_length = adapter_length
+        self.adapter_type = adapter_type
+        self.adapter_weight = adapter_weight
+        self.zero_init_last = zero_init_last
+        self.ln1 = nn.Linear(dim, adapter_length, bias=use_bias)
+        self.activate = act_layer()
+        self.ln2 = nn.Linear(adapter_length, dim, bias=use_bias)
+        self.init_weights()
+        self.init_scaling()
+
+    def _zero_init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            nn.init.zeros_(m.weight)
+            nn.init.zeros_(m.bias)
+
+    def _kaiming_init_weights(self, m):
+        if isinstance(m, nn.Linear):
+            nn.init.kaiming_uniform_(m.weight, a=math.sqrt(5))
+
+    def init_weights(self):
+        self._kaiming_init_weights(self.ln1)
+        if self.zero_init_last:
+            self._zero_init_weights(self.ln2)
+        else:
+            self._kaiming_init_weights(self.ln2)
+
+    def init_scaling(self):
+        if self.adapter_weight:
+            self.scaling = choose_weight_type(self.adapter_weight, self.dim)
+        else:
+            self.scaling = None
+
+    def forward(self, x, x_shortcut=None, use_shortcut=True, **kwargs):
+        if x_shortcut is None:
+            x_shortcut = x
+        x_shape = x.shape
+        if len(x_shape) == 4:
+            b, d, h, w = x_shape
+            x = x.permute(0, 2, 3, 1).reshape(b, h * w, d)
+        out = self.ln2(self.activate(self.ln1(x)))
+        if self.adapter_weight:
+            scaling = get_weight_value(self.adapter_weight, self.scaling, out)
+            out = out * scaling if scaling is not None else out
+        if len(x_shape) == 4:
+            b, d, h, w = x_shape
+            out = out.reshape(b, h, w, -1).permute(0, 3, 1, 2).contiguous()
+        if use_shortcut:
+            out = x_shortcut + out
+        return out

ms-swift/swift/tuners/side.py

@@ -0,0 +1,245 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import copy
+import re
+import types
+from collections import OrderedDict
+from dataclasses import dataclass, field
+from functools import partial
+from itertools import repeat
+from typing import Union
+
+import torch
+from torch import nn
+
+from swift.utils.logger import get_logger
+from swift.utils.torch_utils import find_sub_module
+from .utils import ActivationMixin, SwiftAdapter, SwiftConfig, SwiftOutput
+
+logger = get_logger()
+
+
+@dataclass
+class SideConfig(SwiftConfig):
+    """
+    The configuration class for the side module.
+
+    Side-Tuning only needs to train one side network and
+    weights the output of pre-trained model and side network.
+    'Side-Tuning: A Baseline for Network Adaptation via Additive Side Networks'
+    by Zhang et al.(2019)
+    See https://arxiv.org/abs/1912.13503
+
+    Args:
+        target_modules: The feedforward module to be replaced, in regex format
+    """
+
+    dim: int = field(default=None, metadata={'help': 'The dimension of the hidden states'})
+
+    target_modules: str = field(
+        default=None, metadata={'help': 'The target module to be replaced, in full match format'})
+
+    side_module_name: str = field(default='fcn4', metadata={'help': 'The name of the additive side networks'})
+
+    source_hidden_pos: Union[str, int] = field(
+        default=0,
+        metadata={
+            'help': 'The position of the hidden state input to the target module, can be int (args) or str (kwargs)'
+        })
+
+    target_hidden_pos: Union[str, int] = field(
+        default=0,
+        metadata={
+            'help': 'The position of the hidden state output from the target module, can be int (args) or str (kwargs)'
+        })
+
+    def __post_init__(self):
+        from .mapping import SwiftTuners
+        self.swift_type = SwiftTuners.SIDE
+
+
+class Side(SwiftAdapter):
+
+    @staticmethod
+    def prepare_model(model: nn.Module, config: SideConfig, adapter_name: str) -> SwiftOutput:
+        """Prepare a model with `SideConfig`"""
+        module_keys = [key for key, _ in model.named_modules()]
+
+        for module_key in module_keys:
+            if re.fullmatch(config.target_modules, module_key):  # noqa
+                tgt_module = model.get_submodule(module_key)
+                logger.info(f'Matching target module [{module_key}] of type {type(tgt_module)}')
+                if isinstance(tgt_module, (nn.ModuleList, nn.ModuleDict)):
+                    raise Exception(
+                        f'Type of {type(tgt_module)} may not be supported because of its customized forward')
+
+                def _forward(self, *args, **kwargs):
+                    args_main = getattr(self, f'forward_origin_{adapter_name}')(*args, **kwargs)
+
+                    if isinstance(config.source_hidden_pos, int):
+                        x = args[config.source_hidden_pos]
+                    else:
+                        x = kwargs[config.source_hidden_pos]
+
+                    x_main = args_main[config.target_hidden_pos] \
+                        if isinstance(args_main, (tuple, list, dict)) else args_main
+                    out = getattr(self, f'side_{adapter_name}')(x, x_main)
+                    if isinstance(args_main, (tuple, list, dict)):
+                        args_main[config.target_hidden_pos] = out
+                    else:
+                        args_main = out
+                    return args_main
+
+                if isinstance(tgt_module, nn.Sequential) and not hasattr(tgt_module, 'tgt_module_keys'):
+                    tgt_module.tgt_module_keys = copy.deepcopy(list(tgt_module._modules.keys()))
+
+                    def forward_seq(self, input, *args, **kwargs):
+                        for idx, module in enumerate(self):
+                            if idx >= len(tgt_module.tgt_module_keys):
+                                continue
+                            input = module(input)
+                        return input
+
+                    setattr(tgt_module, f'forward_origin_{adapter_name}', types.MethodType(forward_seq, tgt_module))
+                else:
+                    setattr(tgt_module, f'forward_origin_{adapter_name}', tgt_module.forward)
+                tgt_module.forward = types.MethodType(_forward, tgt_module)
+                side_module = SideModule(config.dim, adapter_name, module_key, config.side_module_name)
+                setattr(tgt_module, f'side_{adapter_name}', side_module)
+                logger.info(f'Side modules(module_key): {module_key}.side_{adapter_name}')
+
+        def state_dict_callback(state_dict, adapter_name, **kwargs):
+            return {key: value for key, value in state_dict.items() if f'side_{adapter_name}' in key}
+
+        def mark_trainable_callback(model):
+            return
+
+        return SwiftOutput(
+            config=config, state_dict_callback=state_dict_callback, mark_trainable_callback=mark_trainable_callback)
+
+    @staticmethod
+    def activate_adapter(module: torch.nn.Module, adapter_name: str, activate: bool, offload: str = None):
+        modules = find_sub_module(module, f'side_{adapter_name}')
+        for _module in modules:
+            _module: ActivationMixin
+            _module: nn.Module
+            _module.set_activation(adapter_name, activate)
+            SwiftAdapter.save_memory(_module, adapter_name, _module.module_key, activate, offload)
+
+
+class SideModule(nn.Module, ActivationMixin):
+    """The implementation of vision side-tuning method.
+
+    Side-Tuning only needs to train one side network and
+    weights the output of pre-trained model and side network.
+    'Side-Tuning: A Baseline for Network Adaptation via Additive Side Networks'
+    by Zhang et al.(2019)
+    See https://arxiv.org/abs/1912.13503
+
+    Args:
+        side_module_name: The name of the additive side networks.
+    """
+
+    def __init__(self, dim, adapter_name, module_key, side_module_name='fcn4'):
+        super(SideModule, self).__init__()
+        super(nn.Module, self).__init__(module_key)
+        self.adapter_name = adapter_name
+
+        side_module_name = side_module_name.lower()
+        if side_module_name == 'fcn4':
+            self.side_net = FCN4(out_dims=dim)
+        elif side_module_name == 'mlp':
+            self.side_net = Mlp(dim)
+        elif side_module_name == 'alexnet':
+            import torchvision
+            mm = torchvision.models.alexnet(pretrained=True)
+            self.side_net = nn.Sequential(
+                OrderedDict([('features', mm.features), ('avgpool', mm.avgpool), ('flatten', nn.Flatten()),
+                             ('fc', nn.Linear(9216, dim, bias=False))]))
+        else:
+            raise ValueError(f'Unsupported side_module_name: {side_module_name}')
+        self.alpha = nn.Parameter(torch.tensor(0.0))
+        self.mark_all_sub_modules_as_plugin()
+
+    def forward(self, x, x_main):
+        if not self.is_activated(self.adapter_name):
+            return x_main
+        alpha_squashed = torch.sigmoid(self.alpha)
+        x_side = self.side_net(x)
+        x_out = alpha_squashed * x_main + (1 - alpha_squashed) * x_side
+        return x_out
+
+
+class FCN4(nn.Module):
+    """The implementation of simple FCN4 network for side network.
+    """
+
+    def __init__(self, out_dims=-1, **kwargs):
+        super(FCN4, self).__init__(**kwargs)
+
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1, bias=False, dilation=1), nn.GroupNorm(2, 16),
+            nn.ReLU())
+        self.conv2 = nn.Sequential(
+            nn.Conv2d(16, 16, kernel_size=3, stride=2, padding=0, bias=False, dilation=1), nn.GroupNorm(2, 16),
+            nn.ReLU())
+        self.conv3 = nn.Sequential(
+            nn.Conv2d(16, 32, kernel_size=3, stride=2, padding=0, bias=False, dilation=1), nn.GroupNorm(2, 32),
+            nn.ReLU())
+        self.conv4 = nn.Sequential(
+            nn.Conv2d(32, 64, kernel_size=3, stride=1, padding=0, bias=False, dilation=1), nn.GroupNorm(2, 64),
+            nn.ReLU())
+        self.pool = nn.AdaptiveAvgPool2d((1, 1))
+        if out_dims > 0:
+            self.fc = nn.Linear(64, out_dims)
+        else:
+            self.fc = None
+
+    def forward(self, x):
+        x = self.conv1(x)
+        x = self.conv2(x)
+        x = self.conv3(x)
+        x = self.conv4(x)
+        x = self.pool(x)
+        x = x.view(x.size(0), -1)
+        if self.fc is not None:
+            x = self.fc(x)
+        return x
+
+
+class Mlp(nn.Module):
+    """ MLP as used in Vision Transformer.
+    """
+
+    def __init__(
+        self,
+        in_features,
+        hidden_features=None,
+        out_features=None,
+        act_layer=nn.GELU,
+        norm_layer=None,
+        bias=True,
+        drop=0.,
+        use_conv=False,
+    ):
+        super().__init__()
+        out_features = out_features or in_features
+        hidden_features = hidden_features or in_features
+        bias = tuple(repeat(bias, 2))
+        drop_probs = tuple(repeat(drop, 2))
+        linear_layer = partial(nn.Conv2d, kernel_size=1) if use_conv else nn.Linear
+
+        self.fc1 = linear_layer(in_features, hidden_features, bias=bias[0])
+        self.act = act_layer()
+        self.drop1 = nn.Dropout(drop_probs[0])
+        self.norm = norm_layer(hidden_features) if norm_layer is not None else nn.Identity()
+        self.fc2 = linear_layer(hidden_features, out_features, bias=bias[1])
+        self.drop2 = nn.Dropout(drop_probs[1])
+
+    def forward(self, x):
+        x = self.fc1(x)
+        x = self.act(x)
+        x = self.drop1(x)
+        x = self.norm(x)
+        x = self.fc2(x)
+        x = self.drop2(x)
+        return x

ms-swift/swift/ui/app.py

@@ -0,0 +1,92 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+from functools import partial
+from typing import List, Union
+
+import gradio as gr
+from packaging import version
+from transformers.utils import strtobool
+
+import swift
+from swift.llm import DeployArguments, EvalArguments, ExportArguments, RLHFArguments, SwiftPipeline, WebUIArguments
+from swift.ui.llm_eval.llm_eval import LLMEval
+from swift.ui.llm_export.llm_export import LLMExport
+from swift.ui.llm_infer.llm_infer import LLMInfer
+from swift.ui.llm_train.llm_train import LLMTrain
+
+locale_dict = {
+    'title': {
+        'zh': '🚀SWIFT: 轻量级大模型训练推理框架',
+        'en': '🚀SWIFT: Scalable lightWeight Infrastructure for Fine-Tuning and Inference'
+    },
+    'sub_title': {
+        'zh':
+        '请查看 <a href=\"https://github.com/modelscope/swift/tree/main/docs/source\" target=\"_blank\">'
+        'SWIFT 文档</a>来查看更多功能，使用SWIFT_UI_LANG=en环境变量来切换英文界面',
+        'en':
+        'Please check <a href=\"https://github.com/modelscope/swift/tree/main/docs/source_en\" target=\"_blank\">'
+        'SWIFT Documentation</a> for more usages, Use SWIFT_UI_LANG=zh variable to switch to Chinese UI',
+    },
+    'star_beggar': {
+        'zh':
+        '喜欢<a href=\"https://github.com/modelscope/swift\" target=\"_blank\">SWIFT</a>就动动手指给我们加个star吧🥺 ',
+        'en':
+        'If you like <a href=\"https://github.com/modelscope/swift\" target=\"_blank\">SWIFT</a>, '
+        'please take a few seconds to star us🥺 '
+    },
+}
+
+
+class SwiftWebUI(SwiftPipeline):
+
+    args_class = WebUIArguments
+    args: args_class
+
+    def run(self):
+        lang = os.environ.get('SWIFT_UI_LANG') or self.args.lang
+        share_env = os.environ.get('WEBUI_SHARE')
+        share = strtobool(share_env) if share_env else self.args.share
+        server = os.environ.get('WEBUI_SERVER') or self.args.server_name
+        port_env = os.environ.get('WEBUI_PORT')
+        port = int(port_env) if port_env else self.args.server_port
+        LLMTrain.set_lang(lang)
+        LLMInfer.set_lang(lang)
+        LLMExport.set_lang(lang)
+        LLMEval.set_lang(lang)
+        with gr.Blocks(title='SWIFT WebUI', theme=gr.themes.Base()) as app:
+            try:
+                _version = swift.__version__
+            except AttributeError:
+                _version = ''
+            gr.HTML(f"<h1><center>{locale_dict['title'][lang]}({_version})</center></h1>")
+            gr.HTML(f"<h3><center>{locale_dict['sub_title'][lang]}</center></h3>")
+            with gr.Tabs():
+                LLMTrain.build_ui(LLMTrain)
+                LLMInfer.build_ui(LLMInfer)
+                LLMExport.build_ui(LLMExport)
+                LLMEval.build_ui(LLMEval)
+
+            concurrent = {}
+            if version.parse(gr.__version__) < version.parse('4.0.0'):
+                concurrent = {'concurrency_count': 5}
+            app.load(
+                partial(LLMTrain.update_input_model, arg_cls=RLHFArguments),
+                inputs=[LLMTrain.element('model')],
+                outputs=[LLMTrain.element('train_record')] + list(LLMTrain.valid_elements().values()))
+            app.load(
+                partial(LLMInfer.update_input_model, arg_cls=DeployArguments, has_record=False),
+                inputs=[LLMInfer.element('model')],
+                outputs=list(LLMInfer.valid_elements().values()))
+            app.load(
+                partial(LLMExport.update_input_model, arg_cls=ExportArguments, has_record=False),
+                inputs=[LLMExport.element('model')],
+                outputs=list(LLMExport.valid_elements().values()))
+            app.load(
+                partial(LLMEval.update_input_model, arg_cls=EvalArguments, has_record=False),
+                inputs=[LLMEval.element('model')],
+                outputs=list(LLMEval.valid_elements().values()))
+        app.queue(**concurrent).launch(server_name=server, inbrowser=True, server_port=port, height=800, share=share)
+
+
+def webui_main(args: Union[List[str], WebUIArguments, None] = None):
+    return SwiftWebUI(args).main()

ms-swift/swift/ui/base.py

@@ -0,0 +1,388 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import dataclasses
+import os
+import sys
+import time
+import typing
+from collections import OrderedDict
+from dataclasses import fields
+from datetime import datetime
+from functools import wraps
+from typing import Any, Dict, List, Type
+
+import gradio as gr
+import json
+from gradio import Accordion, Audio, Button, Checkbox, Dropdown, File, Image, Slider, Tab, TabItem, Textbox, Video
+from modelscope.hub.utils.utils import get_cache_dir
+
+from swift.llm import TEMPLATE_MAPPING, BaseArguments, get_matched_model_meta
+
+all_langs = ['zh', 'en']
+builder: Type['BaseUI'] = None
+base_builder: Type['BaseUI'] = None
+
+
+def update_data(fn):
+
+    @wraps(fn)
+    def wrapper(*args, **kwargs):
+        elem_id = kwargs.get('elem_id', None)
+        self = args[0]
+
+        if builder is not None:
+            choices = base_builder.choice(elem_id)
+            if choices:
+                choices = [str(choice) if choice is not None else None for choice in choices]
+                kwargs['choices'] = choices
+
+        if not isinstance(self, (Tab, TabItem, Accordion)) and 'interactive' not in kwargs:  # noqa
+            kwargs['interactive'] = True
+
+        if 'is_list' in kwargs:
+            self.is_list = kwargs.pop('is_list')
+
+        if base_builder and base_builder.default(elem_id) is not None and not kwargs.get('value'):
+            kwargs['value'] = base_builder.default(elem_id)
+
+        if builder is not None:
+            if elem_id in builder.locales(builder.lang):
+                values = builder.locale(elem_id, builder.lang)
+                if 'info' in values:
+                    kwargs['info'] = values['info']
+                if 'value' in values:
+                    kwargs['value'] = values['value']
+                if 'label' in values:
+                    kwargs['label'] = values['label']
+                if hasattr(builder, 'visible'):
+                    kwargs['visible'] = builder.visible
+                argument = base_builder.argument(elem_id)
+                if argument and 'label' in kwargs:
+                    kwargs['label'] = kwargs['label'] + f'({argument})'
+
+        kwargs['elem_classes'] = 'align'
+        ret = fn(self, **kwargs)
+        self.constructor_args.update(kwargs)
+
+        if builder is not None:
+            builder.element_dict[elem_id] = self
+        return ret
+
+    return wrapper
+
+
+Textbox.__init__ = update_data(Textbox.__init__)
+Dropdown.__init__ = update_data(Dropdown.__init__)
+Checkbox.__init__ = update_data(Checkbox.__init__)
+Slider.__init__ = update_data(Slider.__init__)
+TabItem.__init__ = update_data(TabItem.__init__)
+Accordion.__init__ = update_data(Accordion.__init__)
+Button.__init__ = update_data(Button.__init__)
+File.__init__ = update_data(File.__init__)
+Image.__init__ = update_data(Image.__init__)
+Video.__init__ = update_data(Video.__init__)
+Audio.__init__ = update_data(Audio.__init__)
+
+
+class BaseUI:
+
+    choice_dict: Dict[str, List] = {}
+    default_dict: Dict[str, Any] = {}
+    locale_dict: Dict[str, Dict] = {}
+    element_dict: Dict[str, Dict] = {}
+    arguments: Dict[str, str] = {}
+    sub_ui: List[Type['BaseUI']] = []
+    group: str = None
+    lang: str = all_langs[0]
+    int_regex = r'^[-+]?[0-9]+$'
+    float_regex = r'[-+]?(?:\d*\.*\d+)'
+    bool_regex = r'^(T|t)rue$|^(F|f)alse$'
+    cache_dir = os.path.join(get_cache_dir(), 'swift-web-ui')
+    os.makedirs(cache_dir, exist_ok=True)
+    quote = '\'' if sys.platform != 'win32' else '"'
+    visible = True
+    _locale = {
+        'local_dir_alert': {
+            'value': {
+                'zh': '无法识别model_type和template,请手动选择',
+                'en': 'Cannot recognize the model_type and template, please choose manually'
+            }
+        },
+    }
+
+    @classmethod
+    def build_ui(cls, base_tab: Type['BaseUI']):
+        """Build UI"""
+        global builder, base_builder
+        cls.element_dict = {}
+        old_builder = builder
+        old_base_builder = base_builder
+        builder = cls
+        base_builder = base_tab
+        cls.do_build_ui(base_tab)
+        builder = old_builder
+        base_builder = old_base_builder
+        if cls is base_tab:
+            for ui in cls.sub_ui:
+                ui.after_build_ui(base_tab)
+
+    @classmethod
+    def after_build_ui(cls, base_tab: Type['BaseUI']):
+        pass
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        """Build UI"""
+        pass
+
+    @classmethod
+    def save_cache(cls, key, value):
+        timestamp = str(int(time.time()))
+        key = key.replace('/', '-')
+        filename = os.path.join(cls.cache_dir, key + '-' + timestamp)
+        with open(filename, 'w', encoding='utf-8') as f:
+            json.dump(value, f)
+
+    @classmethod
+    def list_cache(cls, key):
+        files = []
+        key = key.replace('/', '-')
+        for _, _, filenames in os.walk(cls.cache_dir):
+            for filename in filenames:
+                if filename.startswith(key):
+                    idx = filename.rfind('-')
+                    key, ts = filename[:idx], filename[idx + 1:]
+                    dt_object = datetime.fromtimestamp(int(ts))
+                    formatted_time = dt_object.strftime('%Y/%m/%d %H:%M:%S')
+                    files.append(formatted_time)
+        return sorted(files, reverse=True)
+
+    @classmethod
+    def load_cache(cls, key, timestamp) -> BaseArguments:
+        dt_object = datetime.strptime(timestamp, '%Y/%m/%d %H:%M:%S')
+        timestamp = int(dt_object.timestamp())
+        key = key.replace('/', '-')
+        filename = key + '-' + str(timestamp)
+        with open(os.path.join(cls.cache_dir, filename), 'r', encoding='utf-8') as f:
+            return json.load(f)
+
+    @classmethod
+    def clear_cache(cls, key):
+        key = key.replace('/', '-')
+        for _, _, filenames in os.walk(cls.cache_dir):
+            for filename in filenames:
+                if filename.startswith(key):
+                    os.remove(os.path.join(cls.cache_dir, filename))
+
+    @classmethod
+    def choice(cls, elem_id):
+        """Get choice by elem_id"""
+        for sub_ui in BaseUI.sub_ui:
+            _choice = sub_ui.choice(elem_id)
+            if _choice:
+                return _choice
+        return cls.choice_dict.get(elem_id, [])
+
+    @classmethod
+    def default(cls, elem_id):
+        """Get choice by elem_id"""
+        if elem_id in cls.default_dict:
+            return cls.default_dict.get(elem_id)
+        for sub_ui in BaseUI.sub_ui:
+            _choice = sub_ui.default(elem_id)
+            if _choice:
+                return _choice
+        return None
+
+    @classmethod
+    def locale(cls, elem_id, lang):
+        """Get locale by elem_id"""
+        return cls.locales(lang)[elem_id]
+
+    @classmethod
+    def locales(cls, lang):
+        """Get locale by lang"""
+        locales = OrderedDict()
+        for sub_ui in cls.sub_ui:
+            _locales = sub_ui.locales(lang)
+            locales.update(_locales)
+        for key, value in cls.locale_dict.items():
+            locales[key] = {k: v[lang] for k, v in value.items()}
+        return locales
+
+    @classmethod
+    def elements(cls):
+        """Get all elements"""
+        elements = OrderedDict()
+        elements.update(cls.element_dict)
+        for sub_ui in cls.sub_ui:
+            _elements = sub_ui.elements()
+            elements.update(_elements)
+        return elements
+
+    @classmethod
+    def valid_elements(cls):
+        valid_elements = OrderedDict()
+        elements = cls.elements()
+        for key, value in elements.items():
+            if isinstance(value, (Textbox, Dropdown, Slider, Checkbox)) and key != 'train_record':
+                valid_elements[key] = value
+        return valid_elements
+
+    @classmethod
+    def element_keys(cls):
+        return list(cls.elements().keys())
+
+    @classmethod
+    def valid_element_keys(cls):
+        return [
+            key for key, value in cls.elements().items()
+            if isinstance(value, (Textbox, Dropdown, Slider, Checkbox)) and key != 'train_record'
+        ]
+
+    @classmethod
+    def element(cls, elem_id):
+        """Get element by elem_id"""
+        elements = cls.elements()
+        return elements[elem_id]
+
+    @classmethod
+    def argument(cls, elem_id):
+        """Get argument by elem_id"""
+        return cls.arguments.get(elem_id)
+
+    @classmethod
+    def set_lang(cls, lang):
+        cls.lang = lang
+        for sub_ui in cls.sub_ui:
+            sub_ui.lang = lang
+
+    @staticmethod
+    def get_choices_from_dataclass(dataclass):
+        choice_dict = {}
+        for f in fields(dataclass):
+            default_value = f.default
+            if 'MISSING_TYPE' in str(default_value):
+                default_value = None
+            if 'choices' in f.metadata:
+                choice_dict[f.name] = list(f.metadata['choices'])
+            if 'Literal' in str(f.type) and typing.get_args(f.type):
+                choice_dict[f.name] = list(typing.get_args(f.type))
+            if f.name in choice_dict and default_value not in choice_dict[f.name]:
+                choice_dict[f.name].insert(0, default_value)
+        return choice_dict
+
+    @staticmethod
+    def get_default_value_from_dataclass(dataclass):
+        default_dict = {}
+        for f in fields(dataclass):
+            if f.default.__class__ is dataclasses._MISSING_TYPE:
+                default_dict[f.name] = f.default_factory()
+            else:
+                default_dict[f.name] = f.default
+            if isinstance(default_dict[f.name], list):
+                try:
+                    default_dict[f.name] = ' '.join(default_dict[f.name])
+                except TypeError:
+                    default_dict[f.name] = None
+            if not default_dict[f.name]:
+                default_dict[f.name] = None
+        return default_dict
+
+    @staticmethod
+    def get_argument_names(dataclass):
+        arguments = {}
+        for f in fields(dataclass):
+            arguments[f.name] = f'--{f.name}'
+        return arguments
+
+    @classmethod
+    def update_input_model(cls, model, allow_keys=None, has_record=True, arg_cls=BaseArguments, is_ref_model=False):
+        keys = cls.valid_element_keys()
+        if allow_keys:
+            keys = [key for key in keys if key in allow_keys]
+
+        if not model:
+            ret = [gr.update()] * (len(keys) + int(has_record))
+            if len(ret) == 1:
+                return ret[0]
+            else:
+                return ret
+
+        model_meta = get_matched_model_meta(model)
+        local_args_path = os.path.join(model, 'args.json')
+        if model_meta is None and not os.path.exists(local_args_path):
+            gr.Info(cls._locale['local_dir_alert']['value'][cls.lang])
+            ret = [gr.update()] * (len(keys) + int(has_record))
+            if len(ret) == 1:
+                return ret[0]
+            else:
+                return ret
+
+        if os.path.exists(local_args_path):
+            try:
+                if hasattr(arg_cls, 'resume_from_checkpoint'):
+                    try:
+                        args = arg_cls(resume_from_checkpoint=model, load_data_args=True)
+                    except Exception as e:
+                        if 'using `--model`' in str(e):  # TODO a dirty fix
+                            args = arg_cls(model=model, load_data_args=True)
+                        else:
+                            raise e
+                else:
+                    args = arg_cls(ckpt_dir=model, load_data_args=True)
+            except ValueError:
+                return [gr.update()] * (len(keys) + int(has_record))
+            values = []
+            for key in keys:
+                arg_value = getattr(args, key, None)
+                if arg_value and key != 'model':
+                    if key in ('torch_dtype', 'bnb_4bit_compute_dtype'):
+                        arg_value = str(arg_value).split('.')[1]
+                    if isinstance(arg_value, list) and key != 'dataset':
+                        try:
+                            arg_value = ' '.join(arg_value)
+                        except Exception:
+                            arg_value = None
+                    values.append(gr.update(value=arg_value))
+                else:
+                    values.append(gr.update())
+            ret = [gr.update(choices=[])] * int(has_record) + values
+            if len(ret) == 1:
+                return ret[0]
+            else:
+                return ret
+        else:
+            values = []
+            for key in keys:
+                if key not in ('template', 'model_type', 'ref_model_type', 'system'):
+                    values.append(gr.update())
+                elif key in ('template', 'model_type', 'ref_model_type'):
+                    if key == 'ref_model_type':
+                        if is_ref_model:
+                            values.append(gr.update(value=getattr(model_meta, 'model_type')))
+                        else:
+                            values.append(gr.update())
+                    else:
+                        values.append(gr.update(value=getattr(model_meta, key)))
+                else:
+                    values.append(gr.update(value=TEMPLATE_MAPPING[model_meta.template].default_system))
+
+        if has_record:
+            return [gr.update(choices=cls.list_cache(model))] + values
+        else:
+            if len(values) == 1:
+                return values[0]
+            return values
+
+    @classmethod
+    def update_all_settings(cls, model, train_record, base_tab):
+        if not train_record:
+            return [gr.update()] * len(cls.elements())
+        cache = cls.load_cache(model, train_record)
+        updates = []
+        for key, value in base_tab.valid_elements().items():
+            if key in cache:
+                updates.append(gr.update(value=cache[key]))
+            else:
+                updates.append(gr.update())
+        return updates

ms-swift/swift/ui/llm_eval/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

ms-swift/swift/ui/llm_eval/eval.py

@@ -0,0 +1,130 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+from swift.utils import get_logger
+
+logger = get_logger()
+
+
+class Eval(BaseUI):
+
+    group = 'llm_eval'
+
+    locale_dict = {
+        'eval_backend': {
+            'label': {
+                'zh': '评测后端',
+                'en': 'Eval backend'
+            },
+            'info': {
+                'zh': '选择评测后端',
+                'en': 'Select eval backend'
+            }
+        },
+        'eval_dataset': {
+            'label': {
+                'zh': '评测数据集',
+                'en': 'Evaluation dataset'
+            },
+            'info': {
+                'zh': '选择评测数据集，支持多选 (先选择评测后端)',
+                'en': 'Select eval dataset, multiple datasets supported (select eval backend first)'
+            }
+        },
+        'eval_limit': {
+            'label': {
+                'zh': '评测数据个数',
+                'en': 'Eval numbers for each dataset'
+            },
+            'info': {
+                'zh': '每个评测集的取样数',
+                'en': 'Number of rows sampled from each dataset'
+            }
+        },
+        'eval_output_dir': {
+            'label': {
+                'zh': '评测输出目录',
+                'en': 'Eval output dir'
+            },
+            'info': {
+                'zh': '评测结果的输出目录',
+                'en': 'The dir to save the eval results'
+            }
+        },
+        'custom_eval_config': {
+            'label': {
+                'zh': '自定义数据集评测配置',
+                'en': 'Custom eval config'
+            },
+            'info': {
+                'zh': '可以使用该配置评测自己的数据集，详见github文档的评测部分',
+                'en': 'Use this config to eval your own datasets, check the docs in github for details'
+            }
+        },
+        'eval_url': {
+            'label': {
+                'zh': '评测链接',
+                'en': 'The eval url'
+            },
+            'info': {
+                'zh':
+                'OpenAI样式的评测链接(如：http://localhost:8080/v1/chat/completions)，用于评测接口（模型类型输入为实际模型类型）',
+                'en':
+                'The OpenAI style link(like: http://localhost:8080/v1/chat/completions) for '
+                'evaluation(Input actual model type into model_type)'
+            }
+        },
+        'api_key': {
+            'label': {
+                'zh': '接口token',
+                'en': 'The url token'
+            },
+            'info': {
+                'zh': 'eval_url的token',
+                'en': 'The token used with eval_url'
+            }
+        },
+        'infer_backend': {
+            'label': {
+                'zh': '推理框架',
+                'en': 'Infer backend'
+            },
+        }
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        try:
+            from swift.llm.argument.eval_args import EvalArguments
+            eval_dataset_dict = EvalArguments.list_eval_dataset()
+            default_backend = EvalArguments.eval_backend
+        except Exception as e:
+            logger.warn(e)
+            eval_dataset_dict = {}
+            default_backend = None
+
+        with gr.Row():
+            gr.Dropdown(elem_id='eval_backend', choices=list(eval_dataset_dict.keys()), value=default_backend, scale=20)
+            gr.Dropdown(
+                elem_id='eval_dataset',
+                is_list=True,
+                choices=eval_dataset_dict.get(default_backend, []),
+                multiselect=True,
+                allow_custom_value=True,
+                scale=20)
+            gr.Textbox(elem_id='eval_limit', scale=20)
+            gr.Dropdown(elem_id='infer_backend', scale=20)
+        with gr.Row():
+            gr.Textbox(elem_id='custom_eval_config', scale=20)
+            gr.Textbox(elem_id='eval_output_dir', scale=20)
+            gr.Textbox(elem_id='eval_url', scale=20)
+            gr.Textbox(elem_id='api_key', scale=20)
+
+        def update_eval_dataset(backend):
+            return gr.update(choices=eval_dataset_dict[backend])
+
+        cls.element('eval_backend').change(update_eval_dataset, [cls.element('eval_backend')],
+                                           [cls.element('eval_dataset')])

ms-swift/swift/ui/llm_eval/model.py

@@ -0,0 +1,78 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from functools import partial
+from typing import Type
+
+import gradio as gr
+
+from swift.llm import TEMPLATE_MAPPING, EvalArguments, ModelType
+from swift.llm.model.register import get_all_models
+from swift.ui.base import BaseUI
+
+
+class Model(BaseUI):
+
+    group = 'llm_eval'
+
+    locale_dict = {
+        'checkpoint': {
+            'value': {
+                'zh': '训练后的模型',
+                'en': 'Trained model'
+            }
+        },
+        'model_type': {
+            'label': {
+                'zh': '选择模型类型',
+                'en': 'Select Model Type'
+            },
+            'info': {
+                'zh': 'SWIFT已支持的模型类型',
+                'en': 'Base model type supported by SWIFT'
+            }
+        },
+        'model': {
+            'label': {
+                'zh': '模型id或路径',
+                'en': 'Model id or path'
+            },
+            'info': {
+                'zh': '实际的模型id，如果是训练后的模型请填入checkpoint-xxx的目录',
+                'en': 'The actual model id or path, if is a trained model, please fill in the checkpoint-xxx dir'
+            }
+        },
+        'reset': {
+            'value': {
+                'zh': '恢复初始值',
+                'en': 'Reset to default'
+            },
+        },
+        'template': {
+            'label': {
+                'zh': '模型Prompt模板类型',
+                'en': 'Prompt template type'
+            },
+            'info': {
+                'zh': '选择匹配模型的Prompt模板',
+                'en': 'Choose the template type of the model'
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Row():
+            gr.Dropdown(
+                elem_id='model',
+                scale=20,
+                choices=get_all_models(),
+                value='Qwen/Qwen2.5-7B-Instruct',
+                allow_custom_value=True)
+            gr.Dropdown(elem_id='model_type', choices=ModelType.get_model_name_list(), scale=20)
+            gr.Dropdown(elem_id='template', choices=list(TEMPLATE_MAPPING.keys()), scale=20)
+
+    @classmethod
+    def after_build_ui(cls, base_tab: Type['BaseUI']):
+        cls.element('model').change(
+            partial(cls.update_input_model, arg_cls=EvalArguments, has_record=False),
+            inputs=[cls.element('model')],
+            outputs=list(cls.valid_elements().values()))

ms-swift/swift/ui/llm_export/llm_export.py

@@ -0,0 +1,191 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import re
+import sys
+import time
+from datetime import datetime
+from functools import partial
+from typing import Type
+
+import gradio as gr
+import json
+import torch
+from json import JSONDecodeError
+from transformers.utils import is_torch_cuda_available, is_torch_npu_available
+
+from swift.llm import ExportArguments
+from swift.ui.base import BaseUI
+from swift.ui.llm_export.export import Export
+from swift.ui.llm_export.model import Model
+from swift.ui.llm_export.runtime import ExportRuntime
+from swift.utils import get_device_count
+
+
+class LLMExport(BaseUI):
+    group = 'llm_export'
+
+    sub_ui = [Model, Export, ExportRuntime]
+
+    locale_dict = {
+        'llm_export': {
+            'label': {
+                'zh': 'LLM导出',
+                'en': 'LLM export',
+            }
+        },
+        'more_params': {
+            'label': {
+                'zh': '更多参数',
+                'en': 'More params'
+            },
+            'info': {
+                'zh': '以json格式或--xxx xxx命令行格式填入',
+                'en': 'Fill in with json format or --xxx xxx cmd format'
+            }
+        },
+        'export': {
+            'value': {
+                'zh': '开始导出',
+                'en': 'Begin Export'
+            },
+        },
+        'gpu_id': {
+            'label': {
+                'zh': '选择可用GPU',
+                'en': 'Choose GPU'
+            },
+            'info': {
+                'zh': '选择使用的GPU号，如CUDA不可用只能选择CPU',
+                'en': 'Select GPU to export'
+            }
+        },
+    }
+
+    choice_dict = BaseUI.get_choices_from_dataclass(ExportArguments)
+    default_dict = BaseUI.get_default_value_from_dataclass(ExportArguments)
+    arguments = BaseUI.get_argument_names(ExportArguments)
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.TabItem(elem_id='llm_export', label=''):
+            default_device = 'cpu'
+            device_count = get_device_count()
+            if device_count > 0:
+                default_device = '0'
+            with gr.Blocks():
+                Model.build_ui(base_tab)
+                Export.build_ui(base_tab)
+                ExportRuntime.build_ui(base_tab)
+                with gr.Row():
+                    gr.Textbox(elem_id='more_params', lines=4, scale=20)
+                    gr.Button(elem_id='export', scale=2, variant='primary')
+                gr.Dropdown(
+                    elem_id='gpu_id',
+                    multiselect=True,
+                    choices=[str(i) for i in range(device_count)] + ['cpu'],
+                    value=default_device,
+                    scale=8)
+
+                cls.element('export').click(
+                    cls.export_model, list(base_tab.valid_elements().values()),
+                    [cls.element('runtime_tab'), cls.element('running_tasks')])
+
+                base_tab.element('running_tasks').change(
+                    partial(ExportRuntime.task_changed, base_tab=base_tab), [base_tab.element('running_tasks')],
+                    list(base_tab.valid_elements().values()) + [cls.element('log')])
+                ExportRuntime.element('kill_task').click(
+                    ExportRuntime.kill_task,
+                    [ExportRuntime.element('running_tasks')],
+                    [ExportRuntime.element('running_tasks')] + [ExportRuntime.element('log')],
+                )
+
+    @classmethod
+    def export(cls, *args):
+        export_args = cls.get_default_value_from_dataclass(ExportArguments)
+        kwargs = {}
+        kwargs_is_list = {}
+        other_kwargs = {}
+        more_params = {}
+        more_params_cmd = ''
+        keys = cls.valid_element_keys()
+        for key, value in zip(keys, args):
+            compare_value = export_args.get(key)
+            compare_value_arg = str(compare_value) if not isinstance(compare_value, (list, dict)) else compare_value
+            compare_value_ui = str(value) if not isinstance(value, (list, dict)) else value
+            if key in export_args and compare_value_ui != compare_value_arg and value:
+                if isinstance(value, str) and re.fullmatch(cls.int_regex, value):
+                    value = int(value)
+                elif isinstance(value, str) and re.fullmatch(cls.float_regex, value):
+                    value = float(value)
+                elif isinstance(value, str) and re.fullmatch(cls.bool_regex, value):
+                    value = True if value.lower() == 'true' else False
+                kwargs[key] = value if not isinstance(value, list) else ' '.join(value)
+                kwargs_is_list[key] = isinstance(value, list) or getattr(cls.element(key), 'is_list', False)
+            else:
+                other_kwargs[key] = value
+            if key == 'more_params' and value:
+                try:
+                    more_params = json.loads(value)
+                except (JSONDecodeError or TypeError):
+                    more_params_cmd = value
+
+        kwargs.update(more_params)
+        model = kwargs.get('model')
+        if os.path.exists(model) and os.path.exists(os.path.join(model, 'args.json')):
+            kwargs['ckpt_dir'] = kwargs.pop('model')
+        export_args = ExportArguments(
+            **{
+                key: value.split(' ') if key in kwargs_is_list and kwargs_is_list[key] else value
+                for key, value in kwargs.items()
+            })
+        params = ''
+        sep = f'{cls.quote} {cls.quote}'
+        for e in kwargs:
+            if isinstance(kwargs[e], list):
+                params += f'--{e} {cls.quote}{sep.join(kwargs[e])}{cls.quote} '
+            elif e in kwargs_is_list and kwargs_is_list[e]:
+                all_args = [arg for arg in kwargs[e].split(' ') if arg.strip()]
+                params += f'--{e} {cls.quote}{sep.join(all_args)}{cls.quote} '
+            else:
+                params += f'--{e} {cls.quote}{kwargs[e]}{cls.quote} '
+        params += more_params_cmd + ' '
+        devices = other_kwargs['gpu_id']
+        devices = [d for d in devices if d]
+        assert (len(devices) == 1 or 'cpu' not in devices)
+        gpus = ','.join(devices)
+        cuda_param = ''
+        if gpus != 'cpu':
+            if is_torch_npu_available():
+                cuda_param = f'ASCEND_RT_VISIBLE_DEVICES={gpus}'
+            elif is_torch_cuda_available():
+                cuda_param = f'CUDA_VISIBLE_DEVICES={gpus}'
+            else:
+                cuda_param = ''
+        now = datetime.now()
+        time_str = f'{now.year}{now.month}{now.day}{now.hour}{now.minute}{now.second}'
+        file_path = f'output/{export_args.model_type}-{time_str}'
+        if not os.path.exists(file_path):
+            os.makedirs(file_path, exist_ok=True)
+        log_file = os.path.join(os.getcwd(), f'{file_path}/run_export.log')
+        export_args.log_file = log_file
+        params += f'--log_file "{log_file}" '
+        params += '--ignore_args_error true '
+        additional_param = ''
+        if export_args.quant_method == 'gptq':
+            additional_param = 'OMP_NUM_THREADS=14'
+        if sys.platform == 'win32':
+            if cuda_param:
+                cuda_param = f'set {cuda_param} && '
+            if additional_param:
+                additional_param = f'set {additional_param} && '
+            run_command = f'{cuda_param}{additional_param}start /b swift export {params} > {log_file} 2>&1'
+        else:
+            run_command = f'{cuda_param} {additional_param} nohup swift export {params} > {log_file} 2>&1 &'
+        return run_command, export_args, log_file
+
+    @classmethod
+    def export_model(cls, *args):
+        run_command, export_args, log_file = cls.export(*args)
+        os.system(run_command)
+        time.sleep(2)
+        return gr.update(open=True), ExportRuntime.refresh_tasks(log_file)

ms-swift/swift/ui/llm_export/model.py

@@ -0,0 +1,83 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from functools import partial
+from typing import Type
+
+import gradio as gr
+
+from swift.llm import TEMPLATE_MAPPING, ExportArguments, ModelType
+from swift.llm.model.register import get_all_models
+from swift.ui.base import BaseUI
+
+
+class Model(BaseUI):
+
+    group = 'llm_export'
+
+    locale_dict = {
+        'checkpoint': {
+            'value': {
+                'zh': '训练后的模型',
+                'en': 'Trained model'
+            }
+        },
+        'model_type': {
+            'label': {
+                'zh': '选择模型类型',
+                'en': 'Select Model Type'
+            },
+            'info': {
+                'zh': 'SWIFT已支持的模型类型',
+                'en': 'Base model type supported by SWIFT'
+            }
+        },
+        'model': {
+            'label': {
+                'zh': '模型id或路径',
+                'en': 'Model id or path'
+            },
+            'info': {
+                'zh': '实际的模型id，如果是训练后的模型请填入checkpoint-xxx的目录',
+                'en': 'The actual model id or path, if is a trained model, please fill in the checkpoint-xxx dir'
+            }
+        },
+        'reset': {
+            'value': {
+                'zh': '恢复初始值',
+                'en': 'Reset to default'
+            },
+        },
+        'template': {
+            'label': {
+                'zh': '模型Prompt模板类型',
+                'en': 'Prompt template type'
+            },
+            'info': {
+                'zh': '选择匹配模型的Prompt模板',
+                'en': 'Choose the template type of the model'
+            }
+        },
+    }
+
+    ignored_models = ['int1', 'int2', 'int4', 'int8', 'awq', 'gptq', 'bnb', 'eetq', 'aqlm', 'hqq']
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Row():
+            all_models = [
+                model for model in get_all_models() if not any([ignored in model for ignored in cls.ignored_models])
+            ]
+            gr.Dropdown(
+                elem_id='model',
+                scale=20,
+                choices=all_models,
+                value='Qwen/Qwen2.5-7B-Instruct',
+                allow_custom_value=True)
+            gr.Dropdown(elem_id='model_type', choices=ModelType.get_model_name_list(), scale=20)
+            gr.Dropdown(elem_id='template', choices=list(TEMPLATE_MAPPING.keys()), scale=20)
+
+    @classmethod
+    def after_build_ui(cls, base_tab: Type['BaseUI']):
+        cls.element('model').change(
+            partial(cls.update_input_model, arg_cls=ExportArguments, has_record=False),
+            inputs=[cls.element('model')],
+            outputs=list(cls.valid_elements().values()))

ms-swift/swift/ui/llm_export/runtime.py

@@ -0,0 +1,75 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from swift.ui.llm_infer.runtime import Runtime
+from swift.utils import get_logger
+
+logger = get_logger()
+
+
+class ExportRuntime(Runtime):
+
+    group = 'llm_export'
+
+    cmd = 'export'
+
+    locale_dict = {
+        'runtime_tab': {
+            'label': {
+                'zh': '运行时',
+                'en': 'Runtime'
+            },
+        },
+        'running_cmd': {
+            'label': {
+                'zh': '运行命令',
+                'en': 'Command line'
+            },
+            'info': {
+                'zh': '执行的实际命令',
+                'en': 'The actual command'
+            }
+        },
+        'show_log': {
+            'value': {
+                'zh': '展示导出状态',
+                'en': 'Show export status'
+            },
+        },
+        'stop_show_log': {
+            'value': {
+                'zh': '停止展示',
+                'en': 'Stop showing running status'
+            },
+        },
+        'log': {
+            'label': {
+                'zh': '日志输出',
+                'en': 'Logging content'
+            },
+            'info': {
+                'zh': '如果日志无更新请再次点击"展示日志内容"',
+                'en': 'Please press "Show log" if the log content is not updating'
+            }
+        },
+        'running_tasks': {
+            'label': {
+                'zh': '运行中导出任务',
+                'en': 'Running export task'
+            },
+            'info': {
+                'zh': '所有的swift export命令启动的任务',
+                'en': 'All tasks started by swift export'
+            }
+        },
+        'refresh_tasks': {
+            'value': {
+                'zh': '找回导出任务',
+                'en': 'Find export'
+            },
+        },
+        'kill_task': {
+            'value': {
+                'zh': '杀死导出任务',
+                'en': 'Kill export'
+            },
+        },
+    }

ms-swift/swift/ui/llm_infer/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

ms-swift/swift/ui/llm_infer/generate.py

@@ -0,0 +1,65 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+
+
+class Generate(BaseUI):
+
+    group = 'llm_infer'
+
+    locale_dict = {
+        'max_new_tokens': {
+            'label': {
+                'zh': '生成序列最大长度',
+                'en': 'Max new tokens'
+            },
+        },
+        'temperature': {
+            'label': {
+                'zh': 'temperature',
+                'en': 'temperature'
+            },
+        },
+        'top_k': {
+            'label': {
+                'zh': 'top_k',
+                'en': 'top_k'
+            },
+        },
+        'top_p': {
+            'label': {
+                'zh': 'top_p',
+                'en': 'top_p'
+            },
+        },
+        'repetition_penalty': {
+            'label': {
+                'zh': 'repetition_penalty',
+                'en': 'repetition_penalty'
+            },
+        },
+        'system': {
+            'label': {
+                'zh': 'system字段',
+                'en': 'system'
+            },
+            'info': {
+                'zh': 'system字段支持在加载模型后修改',
+                'en': 'system can be modified after the model weights loaded'
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Row():
+            gr.Textbox(elem_id='max_new_tokens', lines=1, value='2048')
+            gr.Slider(elem_id='temperature', minimum=0.0, maximum=10, step=0.1, value=0.3)
+            gr.Slider(elem_id='top_k', minimum=1, maximum=100, step=5, value=20)
+            gr.Slider(elem_id='top_p', minimum=0.0, maximum=1.0, step=0.05, value=0.7)
+            gr.Slider(elem_id='repetition_penalty', minimum=0.0, maximum=10, step=0.05, value=1.05)
+        with gr.Row():
+            gr.Textbox(elem_id='system', lines=4, scale=20)

ms-swift/swift/ui/llm_infer/llm_infer.py

@@ -0,0 +1,396 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import re
+import signal
+import sys
+import time
+from copy import deepcopy
+from datetime import datetime
+from functools import partial
+from typing import List, Type
+
+import gradio as gr
+import json
+import torch
+from json import JSONDecodeError
+from transformers.utils import is_torch_cuda_available, is_torch_npu_available
+
+from swift.llm import DeployArguments, InferArguments, InferClient, InferRequest, RequestConfig
+from swift.ui.base import BaseUI
+from swift.ui.llm_infer.model import Model
+from swift.ui.llm_infer.runtime import Runtime
+from swift.utils import get_device_count, get_logger
+
+logger = get_logger()
+
+
+class LLMInfer(BaseUI):
+
+    group = 'llm_infer'
+
+    is_multimodal = True
+
+    sub_ui = [Model, Runtime]
+
+    locale_dict = {
+        'generate_alert': {
+            'value': {
+                'zh': '请先部署模型',
+                'en': 'Please deploy model first',
+            }
+        },
+        'port': {
+            'label': {
+                'zh': '端口',
+                'en': 'port'
+            },
+        },
+        'llm_infer': {
+            'label': {
+                'zh': 'LLM推理',
+                'en': 'LLM Inference',
+            }
+        },
+        'load_alert': {
+            'value': {
+                'zh': '部署中，请点击"展示部署状态"查看',
+                'en': 'Start to deploy model, '
+                'please Click "Show running '
+                'status" to view details',
+            }
+        },
+        'loaded_alert': {
+            'value': {
+                'zh': '模型加载完成',
+                'en': 'Model loaded'
+            }
+        },
+        'port_alert': {
+            'value': {
+                'zh': '该端口已被占用',
+                'en': 'The port has been occupied'
+            }
+        },
+        'chatbot': {
+            'value': {
+                'zh': '对话框',
+                'en': 'Chat bot'
+            },
+        },
+        'infer_model_type': {
+            'label': {
+                'zh': 'Lora模块',
+                'en': 'Lora module'
+            },
+            'info': {
+                'zh': '发送给server端哪个LoRA，默认为`default`',
+                'en': 'Which LoRA to use on server, default value is `default`'
+            }
+        },
+        'prompt': {
+            'label': {
+                'zh': '请输入：',
+                'en': 'Input:'
+            },
+        },
+        'clear_history': {
+            'value': {
+                'zh': '清除对话信息',
+                'en': 'Clear history'
+            },
+        },
+        'submit': {
+            'value': {
+                'zh': '🚀 发送',
+                'en': '🚀 Send'
+            },
+        },
+        'gpu_id': {
+            'label': {
+                'zh': '选择可用GPU',
+                'en': 'Choose GPU'
+            },
+            'info': {
+                'zh': '选择训练使用的GPU号，如CUDA不可用只能选择CPU',
+                'en': 'Select GPU to train'
+            }
+        },
+    }
+
+    choice_dict = BaseUI.get_choices_from_dataclass(InferArguments)
+    default_dict = BaseUI.get_default_value_from_dataclass(InferArguments)
+    arguments = BaseUI.get_argument_names(InferArguments)
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.TabItem(elem_id='llm_infer', label=''):
+            default_device = 'cpu'
+            device_count = get_device_count()
+            if device_count > 0:
+                default_device = '0'
+            with gr.Blocks():
+                infer_request = gr.State(None)
+                Model.build_ui(base_tab)
+                Runtime.build_ui(base_tab)
+                with gr.Row():
+                    gr.Dropdown(
+                        elem_id='gpu_id',
+                        multiselect=True,
+                        choices=[str(i) for i in range(device_count)] + ['cpu'],
+                        value=default_device,
+                        scale=8)
+                    infer_model_type = gr.Textbox(elem_id='infer_model_type', scale=4)
+                    gr.Textbox(elem_id='port', lines=1, value='8000', scale=4)
+                chatbot = gr.Chatbot(elem_id='chatbot', elem_classes='control-height')
+                with gr.Row():
+                    prompt = gr.Textbox(elem_id='prompt', lines=1, interactive=True)
+                    with gr.Tabs(visible=cls.is_multimodal):
+                        with gr.TabItem(label='Image'):
+                            image = gr.Image(type='filepath')
+                        with gr.TabItem(label='Video'):
+                            video = gr.Video()
+                        with gr.TabItem(label='Audio'):
+                            audio = gr.Audio(type='filepath')
+
+                with gr.Row():
+                    clear_history = gr.Button(elem_id='clear_history')
+                    submit = gr.Button(elem_id='submit')
+
+                cls.element('load_checkpoint').click(
+                    cls.deploy_model, list(base_tab.valid_elements().values()),
+                    [cls.element('runtime_tab'), cls.element('running_tasks')])
+                submit.click(
+                    cls.send_message,
+                    inputs=[
+                        cls.element('running_tasks'),
+                        cls.element('template'), prompt, image, video, audio, infer_request, infer_model_type,
+                        cls.element('system'),
+                        cls.element('max_new_tokens'),
+                        cls.element('temperature'),
+                        cls.element('top_k'),
+                        cls.element('top_p'),
+                        cls.element('repetition_penalty')
+                    ],
+                    outputs=[prompt, chatbot, image, video, audio, infer_request],
+                    queue=True)
+
+                clear_history.click(
+                    fn=cls.clear_session, inputs=[], outputs=[prompt, chatbot, image, video, audio, infer_request])
+
+                base_tab.element('running_tasks').change(
+                    partial(Runtime.task_changed, base_tab=base_tab), [base_tab.element('running_tasks')],
+                    list(cls.valid_elements().values()) + [cls.element('log')])
+                Runtime.element('kill_task').click(
+                    Runtime.kill_task,
+                    [Runtime.element('running_tasks')],
+                    [Runtime.element('running_tasks')] + [Runtime.element('log')],
+                )
+
+    @classmethod
+    def deploy(cls, *args):
+        deploy_args = cls.get_default_value_from_dataclass(DeployArguments)
+        kwargs = {}
+        kwargs_is_list = {}
+        other_kwargs = {}
+        more_params = {}
+        more_params_cmd = ''
+        keys = cls.valid_element_keys()
+        for key, value in zip(keys, args):
+            compare_value = deploy_args.get(key)
+            compare_value_arg = str(compare_value) if not isinstance(compare_value, (list, dict)) else compare_value
+            compare_value_ui = str(value) if not isinstance(value, (list, dict)) else value
+            if key in deploy_args and compare_value_ui != compare_value_arg and value:
+                if isinstance(value, str) and re.fullmatch(cls.int_regex, value):
+                    value = int(value)
+                elif isinstance(value, str) and re.fullmatch(cls.float_regex, value):
+                    value = float(value)
+                elif isinstance(value, str) and re.fullmatch(cls.bool_regex, value):
+                    value = True if value.lower() == 'true' else False
+                kwargs[key] = value if not isinstance(value, list) else ' '.join(value)
+                kwargs_is_list[key] = isinstance(value, list) or getattr(cls.element(key), 'is_list', False)
+            else:
+                other_kwargs[key] = value
+            if key == 'more_params' and value:
+                try:
+                    more_params = json.loads(value)
+                except (JSONDecodeError or TypeError):
+                    more_params_cmd = value
+
+        kwargs.update(more_params)
+        model = kwargs.get('model')
+        if os.path.exists(model) and os.path.exists(os.path.join(model, 'args.json')):
+            kwargs['ckpt_dir'] = kwargs.pop('model')
+            with open(os.path.join(kwargs['ckpt_dir'], 'args.json'), 'r', encoding='utf-8') as f:
+                _json = json.load(f)
+                kwargs['model_type'] = _json['model_type']
+                kwargs['train_type'] = _json['train_type']
+        deploy_args = DeployArguments(
+            **{
+                key: value.split(' ') if key in kwargs_is_list and kwargs_is_list[key] else value
+                for key, value in kwargs.items()
+            })
+        if deploy_args.port in Runtime.get_all_ports():
+            raise gr.Error(cls.locale('port_alert', cls.lang)['value'])
+        params = ''
+        sep = f'{cls.quote} {cls.quote}'
+        for e in kwargs:
+            if isinstance(kwargs[e], list):
+                params += f'--{e} {cls.quote}{sep.join(kwargs[e])}{cls.quote} '
+            elif e in kwargs_is_list and kwargs_is_list[e]:
+                all_args = [arg for arg in kwargs[e].split(' ') if arg.strip()]
+                params += f'--{e} {cls.quote}{sep.join(all_args)}{cls.quote} '
+            else:
+                params += f'--{e} {cls.quote}{kwargs[e]}{cls.quote} '
+        if 'port' not in kwargs:
+            params += f'--port "{deploy_args.port}" '
+        params += more_params_cmd + ' '
+        devices = other_kwargs['gpu_id']
+        devices = [d for d in devices if d]
+        assert (len(devices) == 1 or 'cpu' not in devices)
+        gpus = ','.join(devices)
+        cuda_param = ''
+        if gpus != 'cpu':
+            if is_torch_npu_available():
+                cuda_param = f'ASCEND_RT_VISIBLE_DEVICES={gpus}'
+            elif is_torch_cuda_available():
+                cuda_param = f'CUDA_VISIBLE_DEVICES={gpus}'
+            else:
+                cuda_param = ''
+        now = datetime.now()
+        time_str = f'{now.year}{now.month}{now.day}{now.hour}{now.minute}{now.second}'
+        file_path = f'output/{deploy_args.model_type}-{time_str}'
+        if not os.path.exists(file_path):
+            os.makedirs(file_path, exist_ok=True)
+        log_file = os.path.join(os.getcwd(), f'{file_path}/run_deploy.log')
+        deploy_args.log_file = log_file
+        params += f'--log_file "{log_file}" '
+        params += '--ignore_args_error true '
+        if sys.platform == 'win32':
+            if cuda_param:
+                cuda_param = f'set {cuda_param} && '
+            run_command = f'{cuda_param}start /b swift deploy {params} > {log_file} 2>&1'
+        else:
+            run_command = f'{cuda_param} nohup swift deploy {params} > {log_file} 2>&1 &'
+        return run_command, deploy_args, log_file
+
+    @classmethod
+    def deploy_model(cls, *args):
+        run_command, deploy_args, log_file = cls.deploy(*args)
+        logger.info(f'Running deployment command: {run_command}')
+        os.system(run_command)
+        gr.Info(cls.locale('load_alert', cls.lang)['value'])
+        time.sleep(2)
+        running_task = Runtime.refresh_tasks(log_file)
+        return gr.update(open=True), running_task
+
+    @classmethod
+    def register_clean_hook(cls):
+        signal.signal(signal.SIGINT, LLMInfer.signal_handler)
+        if os.name != 'nt':
+            signal.signal(signal.SIGTERM, LLMInfer.signal_handler)
+
+    @staticmethod
+    def signal_handler(*args, **kwargs):
+        LLMInfer.clean_deployment()
+        sys.exit(0)
+
+    @classmethod
+    def clear_session(cls):
+        return '', [], gr.update(value=None), gr.update(value=None), gr.update(value=None), []
+
+    @classmethod
+    def _replace_tag_with_media(cls, infer_request: InferRequest):
+        total_history = []
+        messages = deepcopy(infer_request.messages)
+        if messages[0]['role'] == 'system':
+            messages.pop(0)
+        for i in range(0, len(messages), 2):
+            slices = messages[i:i + 2]
+            if len(slices) == 2:
+                user, assistant = slices
+            else:
+                user = slices[0]
+                assistant = {'role': 'assistant', 'content': None}
+            user['content'] = (user['content'] or '').replace('<image>', '').replace('<video>',
+                                                                                     '').replace('<audio>', '').strip()
+            for media in user['medias']:
+                total_history.append([(media, ), None])
+            if user['content'] or assistant['content']:
+                total_history.append((user['content'], assistant['content']))
+        return total_history
+
+    @classmethod
+    def agent_type(cls, response):
+        if not response:
+            return None
+        if response.lower().endswith('observation:'):
+            return 'react'
+        if 'observation:' not in response.lower() and 'action input:' in response.lower():
+            return 'toolbench'
+        return None
+
+    @classmethod
+    def send_message(cls, running_task, template_type, prompt: str, image, video, audio, infer_request: InferRequest,
+                     infer_model_type, system, max_new_tokens, temperature, top_k, top_p, repetition_penalty):
+
+        if not infer_request:
+            infer_request = InferRequest(messages=[])
+        if system:
+            if not infer_request.messages or infer_request.messages[0]['role'] != 'system':
+                infer_request.messages.insert(0, {'role': 'system', 'content': system})
+            else:
+                infer_request.messages[0]['content'] = system
+        if not infer_request.messages or infer_request.messages[-1]['role'] != 'user':
+            infer_request.messages.append({'role': 'user', 'content': '', 'medias': []})
+        media = image or video or audio
+        media_type = 'images' if image else 'videos' if video else 'audios'
+        if media:
+            _saved_medias: List = getattr(infer_request, media_type)
+            if not _saved_medias or _saved_medias[-1] != media:
+                _saved_medias.append(media)
+                infer_request.messages[-1]['content'] = infer_request.messages[-1]['content'] + f'<{media_type[:-1]}>'
+                infer_request.messages[-1]['medias'].append(media)
+
+        if not prompt:
+            yield '', cls._replace_tag_with_media(infer_request), gr.update(value=None), gr.update(
+                value=None), gr.update(value=None), infer_request
+            return
+        else:
+            infer_request.messages[-1]['content'] = infer_request.messages[-1]['content'] + prompt
+
+        _, args = Runtime.parse_info_from_cmdline(running_task)
+        request_config = RequestConfig(
+            temperature=temperature, top_k=top_k, top_p=top_p, repetition_penalty=repetition_penalty)
+        request_config.stream = True
+        request_config.stop = ['Observation:']
+        request_config.max_tokens = max_new_tokens
+        stream_resp_with_history = ''
+        response = ''
+        i = len(infer_request.messages) - 1
+        for i in range(len(infer_request.messages) - 1, -1, -1):
+            if infer_request.messages[i]['role'] == 'assistant':
+                response = infer_request.messages[i]['content']
+        agent_type = cls.agent_type(response)
+        if i != len(infer_request.messages) - 1 and agent_type == 'toolbench':
+            infer_request.messages[i + 1]['role'] = 'tool'
+
+        chat = not template_type.endswith('generation')
+        _infer_request = deepcopy(infer_request)
+        for m in _infer_request.messages:
+            if 'medias' in m:
+                m.pop('medias')
+        model_kwargs = {}
+        if infer_model_type:
+            model_kwargs = {'model': infer_model_type}
+        gen_list = InferClient(
+            port=args['port'], ).infer(
+                infer_requests=[_infer_request], request_config=request_config, **model_kwargs)
+        if infer_request.messages[-1]['role'] != 'assistant':
+            infer_request.messages.append({'role': 'assistant', 'content': ''})
+        for chunk in gen_list[0]:
+            if chunk is None:
+                continue
+            stream_resp_with_history += chunk.choices[0].delta.content if chat else chunk.choices[0].text
+            infer_request.messages[-1]['content'] = stream_resp_with_history
+            yield '', cls._replace_tag_with_media(infer_request), gr.update(value=None), gr.update(
+                value=None), gr.update(value=None), infer_request

ms-swift/swift/ui/llm_infer/model.py

@@ -0,0 +1,126 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from functools import partial
+from typing import Type
+
+import gradio as gr
+
+from swift.llm import TEMPLATE_MAPPING, DeployArguments, ModelType
+from swift.llm.model.register import get_all_models
+from swift.ui.base import BaseUI
+from swift.ui.llm_infer.generate import Generate
+
+
+class Model(BaseUI):
+
+    llm_train = 'llm_infer'
+
+    sub_ui = [Generate]
+
+    locale_dict = {
+        'model_type': {
+            'label': {
+                'zh': '选择模型类型',
+                'en': 'Select Model Type'
+            },
+            'info': {
+                'zh': 'SWIFT已支持的模型类型',
+                'en': 'Base model type supported by SWIFT'
+            }
+        },
+        'load_checkpoint': {
+            'value': {
+                'zh': '部署模型',
+                'en': 'Deploy model',
+            }
+        },
+        'model': {
+            'label': {
+                'zh': '模型id或路径',
+                'en': 'Model id or path'
+            },
+            'info': {
+                'zh': '实际的模型id，如果是训练后的模型请填入checkpoint-xxx的目录',
+                'en': 'The actual model id or path, if is a trained model, please fill in the checkpoint-xxx dir'
+            }
+        },
+        'template': {
+            'label': {
+                'zh': '模型Prompt模板类型',
+                'en': 'Prompt template type'
+            },
+            'info': {
+                'zh': '选择匹配模型的Prompt模板',
+                'en': 'Choose the template type of the model'
+            }
+        },
+        'merge_lora': {
+            'label': {
+                'zh': '合并lora',
+                'en': 'merge lora'
+            },
+            'info': {
+                'zh': '仅在sft_type=lora时可用',
+                'en': 'Only available when sft_type=lora'
+            }
+        },
+        'lora_modules': {
+            'label': {
+                'zh': '外部lora模块',
+                'en': 'More lora modules'
+            },
+            'info': {
+                'zh': '空格分割的name=/path1/path2键值对',
+                'en': 'name=/path1/path2 split by blanks'
+            }
+        },
+        'more_params': {
+            'label': {
+                'zh': '更多参数',
+                'en': 'More params'
+            },
+            'info': {
+                'zh': '以json格式或--xxx xxx命令行格式填入',
+                'en': 'Fill in with json format or --xxx xxx cmd format'
+            }
+        },
+        'reset': {
+            'value': {
+                'zh': '恢复初始值',
+                'en': 'Reset to default'
+            },
+        },
+        'infer_backend': {
+            'label': {
+                'zh': '推理框架',
+                'en': 'Infer backend'
+            },
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Row():
+            gr.Dropdown(
+                elem_id='model',
+                scale=20,
+                choices=get_all_models(),
+                value='Qwen/Qwen2.5-7B-Instruct',
+                allow_custom_value=True)
+            gr.Dropdown(elem_id='model_type', choices=ModelType.get_model_name_list(), scale=20)
+            gr.Dropdown(elem_id='template', choices=list(TEMPLATE_MAPPING.keys()), scale=20)
+            gr.Checkbox(elem_id='merge_lora', scale=4)
+            gr.Button(elem_id='reset', scale=2)
+        with gr.Row():
+            gr.Dropdown(elem_id='infer_backend', value='pt', scale=5)
+        Generate.build_ui(base_tab)
+        with gr.Row():
+            gr.Textbox(elem_id='lora_modules', lines=1, is_list=True, scale=40)
+            gr.Textbox(elem_id='more_params', lines=1, scale=20)
+            gr.Button(elem_id='load_checkpoint', scale=2, variant='primary')
+
+    @classmethod
+    def after_build_ui(cls, base_tab: Type['BaseUI']):
+        cls.element('model').change(
+            partial(cls.update_input_model, arg_cls=DeployArguments, has_record=False),
+            inputs=[cls.element('model')],
+            outputs=list(cls.valid_elements().values()))

ms-swift/swift/ui/llm_infer/runtime.py

@@ -0,0 +1,285 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import collections
+import os.path
+import sys
+import time
+from datetime import datetime
+from typing import Dict, List, Tuple, Type
+
+import gradio as gr
+import psutil
+from packaging import version
+
+from swift.ui.base import BaseUI
+from swift.utils import get_logger
+from swift.utils.utils import format_time
+
+logger = get_logger()
+
+
+class Runtime(BaseUI):
+    handlers: Dict[str, Tuple[List, Tuple]] = {}
+
+    group = 'llm_infer'
+
+    cmd = 'deploy'
+
+    log_event = {}
+
+    locale_dict = {
+        'runtime_tab': {
+            'label': {
+                'zh': '运行时',
+                'en': 'Runtime'
+            },
+        },
+        'running_cmd': {
+            'label': {
+                'zh': '运行命令',
+                'en': 'Command line'
+            },
+            'info': {
+                'zh': '执行的实际命令',
+                'en': 'The actual command'
+            }
+        },
+        'show_log': {
+            'value': {
+                'zh': '展示部署状态',
+                'en': 'Show running status'
+            },
+        },
+        'stop_show_log': {
+            'value': {
+                'zh': '停止展示',
+                'en': 'Stop showing running status'
+            },
+        },
+        'log': {
+            'label': {
+                'zh': '日志输出',
+                'en': 'Logging content'
+            },
+            'info': {
+                'zh': '如果日志无更新请再次点击"展示日志内容"',
+                'en': 'Please press "Show log" if the log content is not updating'
+            }
+        },
+        'running_tasks': {
+            'label': {
+                'zh': '运行中部署',
+                'en': 'Running deployments'
+            },
+            'info': {
+                'zh': '所有的swift deploy命令启动的任务',
+                'en': 'Started by swift deploy'
+            }
+        },
+        'refresh_tasks': {
+            'value': {
+                'zh': '找回部署',
+                'en': 'Find deployments'
+            },
+        },
+        'kill_task': {
+            'value': {
+                'zh': '杀死部署',
+                'en': 'Kill running task'
+            },
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='runtime_tab', open=False, visible=True):
+            with gr.Blocks():
+                with gr.Row():
+                    gr.Dropdown(elem_id='running_tasks', scale=10, allow_custom_value=True)
+                    gr.Button(elem_id='refresh_tasks', scale=1, variant='primary')
+                    gr.Button(elem_id='show_log', scale=1, variant='primary')
+                    gr.Button(elem_id='stop_show_log', scale=1)
+                    gr.Button(elem_id='kill_task', scale=1)
+                with gr.Row():
+                    gr.Textbox(elem_id='log', lines=6, visible=False)
+
+                concurrency_limit = {}
+                if version.parse(gr.__version__) >= version.parse('4.0.0'):
+                    concurrency_limit = {'concurrency_limit': 5}
+                base_tab.element('show_log').click(cls.update_log, [],
+                                                   [cls.element('log')]).then(cls.wait,
+                                                                              [base_tab.element('running_tasks')],
+                                                                              [cls.element('log')], **concurrency_limit)
+
+                base_tab.element('stop_show_log').click(cls.break_log_event, [cls.element('running_tasks')], [])
+
+                base_tab.element('refresh_tasks').click(
+                    cls.refresh_tasks,
+                    [base_tab.element('running_tasks')],
+                    [base_tab.element('running_tasks')],
+                )
+
+    @classmethod
+    def break_log_event(cls, task):
+        if not task:
+            return
+        pid, all_args = cls.parse_info_from_cmdline(task)
+        cls.log_event[all_args['log_file']] = True
+
+    @classmethod
+    def update_log(cls):
+        return gr.update(visible=True)
+
+    @classmethod
+    def wait(cls, task):
+        if not task:
+            return [None]
+        _, args = cls.parse_info_from_cmdline(task)
+        log_file = args['log_file']
+        cls.log_event[log_file] = False
+        offset = 0
+        latest_data = ''
+        lines = collections.deque(maxlen=int(os.environ.get('MAX_LOG_LINES', 50)))
+        try:
+            with open(log_file, 'r', encoding='utf-8') as input:
+                input.seek(offset)
+                fail_cnt = 0
+                while True:
+                    try:
+                        latest_data += input.read()
+                    except UnicodeDecodeError:
+                        continue
+                    if not latest_data:
+                        time.sleep(0.5)
+                        fail_cnt += 1
+                        if fail_cnt > 50:
+                            break
+
+                    if cls.log_event.get(log_file, False):
+                        cls.log_event[log_file] = False
+                        break
+
+                    if '\n' not in latest_data:
+                        continue
+                    latest_lines = latest_data.split('\n')
+                    if latest_data[-1] != '\n':
+                        latest_data = latest_lines[-1]
+                        latest_lines = latest_lines[:-1]
+                    else:
+                        latest_data = ''
+                    lines.extend(latest_lines)
+                    yield '\n'.join(lines)
+        except IOError:
+            pass
+
+    @classmethod
+    def get_all_ports(cls):
+        process_name = 'swift'
+        cmd_name = cls.cmd
+        ports = set()
+        for proc in psutil.process_iter():
+            try:
+                cmdlines = proc.cmdline()
+            except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess):
+                cmdlines = []
+            if any([process_name in cmdline for cmdline in cmdlines]) and any(  # noqa
+                [cmd_name == cmdline for cmdline in cmdlines]):  # noqa
+                try:
+                    ports.add(int(cls.parse_info_from_cmdline(cls.construct_running_task(proc))[1].get('port', 8000)))
+                except IndexError:
+                    pass
+        return ports
+
+    @classmethod
+    def refresh_tasks(cls, running_task=None):
+        log_file = running_task if not running_task or 'pid:' not in running_task else None
+        process_name = 'swift'
+        negative_name = 'swift.exe'
+        cmd_name = cls.cmd
+        process = []
+        selected = None
+        for proc in psutil.process_iter():
+            try:
+                cmdlines = proc.cmdline()
+            except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess):
+                cmdlines = []
+            if any([process_name in cmdline
+                    for cmdline in cmdlines]) and not any([negative_name in cmdline
+                                                           for cmdline in cmdlines]) and any(  # noqa
+                                                               [cmd_name == cmdline for cmdline in cmdlines]):  # noqa
+                process.append(cls.construct_running_task(proc))
+                if log_file is not None and any(  # noqa
+                    [log_file == cmdline for cmdline in cmdlines]):  # noqa
+                    selected = cls.construct_running_task(proc)
+        if not selected:
+            if running_task and running_task in process:
+                selected = running_task
+        if not selected and process:
+            selected = process[0]
+        return gr.update(choices=process, value=selected)
+
+    @staticmethod
+    def construct_running_task(proc):
+        pid = proc.pid
+        ts = time.time()
+        create_time = proc.create_time()
+        create_time_formatted = datetime.fromtimestamp(create_time).strftime('%Y-%m-%d, %H:%M')
+
+        return f'pid:{pid}/create:{create_time_formatted}' \
+               f'/running:{format_time(ts - create_time)}/cmd:{" ".join(proc.cmdline())}'
+
+    @classmethod
+    def parse_info_from_cmdline(cls, task):
+        pid = None
+        for i in range(3):
+            slash = task.find('/')
+            if i == 0:
+                pid = task[:slash].split(':')[1]
+            task = task[slash + 1:]
+        args = task.split(f'swift {cls.cmd}')[1]
+        args = [arg.strip() for arg in args.split('--') if arg.strip()]
+        all_args = {}
+        for i in range(len(args)):
+            space = args[i].find(' ')
+            splits = args[i][:space], args[i][space + 1:]
+            all_args[splits[0]] = splits[1]
+        return pid, all_args
+
+    @classmethod
+    def kill_task(cls, task):
+        if task:
+            pid, all_args = cls.parse_info_from_cmdline(task)
+            log_file = all_args['log_file']
+            if sys.platform == 'win32':
+                os.system(f'taskkill /f /t /pid "{pid}"')
+            else:
+                os.system(f'pkill -9 -f {log_file}')
+            time.sleep(1)
+            cls.break_log_event(task)
+        return [cls.refresh_tasks()] + [gr.update(value=None)]
+
+    @classmethod
+    def task_changed(cls, task, base_tab):
+        if task:
+            _, all_args = cls.parse_info_from_cmdline(task)
+        else:
+            all_args = {}
+        elements = list(base_tab.valid_elements().values())
+        ret = []
+        is_custom_path = 'ckpt_dir' in all_args
+        for e in elements:
+            if e.elem_id in all_args:
+                if isinstance(e, gr.Dropdown) and e.multiselect:
+                    arg = all_args[e.elem_id].split(' ')
+                else:
+                    if e.elem_id == 'model':
+                        if is_custom_path:
+                            arg = all_args['ckpt_dir']
+                        else:
+                            arg = all_args[e.elem_id]
+                    else:
+                        arg = all_args[e.elem_id]
+                ret.append(gr.update(value=arg))
+            else:
+                ret.append(gr.update())
+        cls.break_log_event(task)
+        return ret + [gr.update(value=None)]

ms-swift/swift/ui/llm_train/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

ms-swift/swift/ui/llm_train/advanced.py

@@ -0,0 +1,164 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+
+
+class Advanced(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'advanced_param': {
+            'label': {
+                'zh': '高级参数设置',
+                'en': 'Advanced settings'
+            },
+        },
+        'optim': {
+            'label': {
+                'zh': 'Optimizer类型',
+                'en': 'The Optimizer type'
+            },
+            'info': {
+                'zh': '设置Optimizer类型',
+                'en': 'Set the Optimizer type'
+            }
+        },
+        'weight_decay': {
+            'label': {
+                'zh': '权重衰减',
+                'en': 'Weight decay'
+            },
+            'info': {
+                'zh': '设置weight decay',
+                'en': 'Set the weight decay'
+            }
+        },
+        'logging_steps': {
+            'label': {
+                'zh': '日志打印步数',
+                'en': 'Logging steps'
+            },
+            'info': {
+                'zh': '设置日志打印的步数间隔',
+                'en': 'Set the logging interval'
+            }
+        },
+        'lr_scheduler_type': {
+            'label': {
+                'zh': 'LrScheduler类型',
+                'en': 'The LrScheduler type'
+            },
+            'info': {
+                'zh': '设置LrScheduler类型',
+                'en': 'Set the LrScheduler type'
+            }
+        },
+        'warmup_ratio': {
+            'label': {
+                'zh': '学习率warmup比例',
+                'en': 'Lr warmup ratio'
+            },
+            'info': {
+                'zh': '设置学习率warmup比例',
+                'en': 'Set the warmup ratio in total steps'
+            }
+        },
+        'more_params': {
+            'label': {
+                'zh': '其他高级参数',
+                'en': 'Other params'
+            },
+            'info': {
+                'zh': '以json格式或--xxx xxx命令行格式填入',
+                'en': 'Fill in with json format or --xxx xxx cmd format'
+            }
+        },
+        'truncation_strategy': {
+            'label': {
+                'zh': '数据集超长策略',
+                'en': 'Dataset truncation strategy'
+            },
+            'info': {
+                'zh': '如果token超长该如何处理',
+                'en': 'How to deal with the rows exceed the max length'
+            }
+        },
+        'max_steps': {
+            'label': {
+                'zh': '最大迭代步数',
+                'en': 'Max steps',
+            },
+            'info': {
+                'zh': '设置最大迭代步数，该值如果大于零则数据集迭代次数不生效',
+                'en': 'Set the max steps, if the value > 0 then num_train_epochs has no effects',
+            }
+        },
+        'per_device_eval_batch_size': {
+            'label': {
+                'zh': '验证batch size',
+                'en': 'Val batch size',
+            },
+            'info': {
+                'zh': '验证的batch size',
+                'en': 'Set the val batch size',
+            }
+        },
+        'max_grad_norm': {
+            'label': {
+                'zh': '梯度裁剪',
+                'en': 'Max grad norm',
+            },
+            'info': {
+                'zh': '设置梯度裁剪',
+                'en': 'Set the max grad norm',
+            }
+        },
+        'predict_with_generate': {
+            'label': {
+                'zh': '使用生成指标代替loss',
+                'en': 'Use generate metric instead of loss',
+            },
+            'info': {
+                'zh': '验证时使用generate/Rouge代替loss',
+                'en': 'Use model.generate/Rouge instead of loss',
+            }
+        },
+        'deepspeed': {
+            'label': {
+                'zh': 'deepspeed',
+                'en': 'deepspeed',
+            },
+            'info': {
+                'zh': '可以选择下拉列表，也支持传入路径',
+                'en': 'Choose from the dropbox or fill in a valid path',
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='advanced_param', open=False):
+            with gr.Blocks():
+                with gr.Row():
+                    gr.Textbox(elem_id='optim', lines=1, scale=20)
+                    gr.Textbox(elem_id='weight_decay', lines=1, scale=20)
+                    gr.Textbox(elem_id='logging_steps', lines=1, scale=20)
+                    gr.Textbox(elem_id='lr_scheduler_type', lines=1, scale=20)
+                    gr.Textbox(elem_id='max_steps', lines=1, scale=20)
+                    gr.Slider(elem_id='warmup_ratio', minimum=0.0, maximum=1.0, step=0.05, scale=20)
+                with gr.Row():
+                    gr.Dropdown(elem_id='truncation_strategy', scale=20)
+                    gr.Slider(elem_id='per_device_eval_batch_size', minimum=1, maximum=256, step=2, scale=20)
+                    gr.Textbox(elem_id='max_grad_norm', lines=1, scale=20)
+                    gr.Dropdown(
+                        elem_id='deepspeed',
+                        scale=20,
+                        allow_custom_value=True,
+                        value=None,
+                        choices=['zero0', 'zero1', 'zero2', 'zero3', 'zero2_offload', 'zero3_offload'])
+                with gr.Row():
+                    gr.Textbox(elem_id='more_params', lines=4, scale=20)

ms-swift/swift/ui/llm_train/dataset.py

@@ -0,0 +1,91 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.llm.dataset.register import get_dataset_list
+from swift.ui.base import BaseUI
+
+
+class Dataset(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'dataset': {
+            'label': {
+                'zh': '数据集名称',
+                'en': 'Dataset Code'
+            },
+            'info': {
+                'zh': '选择训练的数据集，支持复选/本地路径',
+                'en': 'The dataset(s) to train the models, support multi select and local folder/files'
+            }
+        },
+        'max_length': {
+            'label': {
+                'zh': '句子最大长度',
+                'en': 'The max length',
+            },
+            'info': {
+                'zh': '设置输入模型的最大长度',
+                'en': 'Set the max length input to the model',
+            }
+        },
+        'split_dataset_ratio': {
+            'label': {
+                'zh': '验证集拆分比例',
+                'en': 'Split ratio of eval dataset'
+            },
+            'info': {
+                'zh': '表示将总数据的多少拆分到验证集中',
+                'en': 'Split the datasets by this ratio for eval'
+            }
+        },
+        'train_dataset_sample': {
+            'label': {
+                'zh': '训练集采样数量',
+                'en': 'The sample size from the train dataset'
+            },
+            'info': {
+                'zh': '从训练集中采样一定行数进行训练',
+                'en': 'Train with the sample size from the dataset',
+            }
+        },
+        'val_dataset_sample': {
+            'label': {
+                'zh': '验证集采样数量',
+                'en': 'The sample size from the val dataset'
+            },
+            'info': {
+                'zh': '从验证集中采样一定行数进行训练',
+                'en': 'Validate with the sample size from the dataset',
+            }
+        },
+        'custom_dataset_info': {
+            'label': {
+                'zh': '外部数据集配置',
+                'en': 'Custom dataset config'
+            },
+            'info': {
+                'zh': '注册外部数据集的配置文件',
+                'en': 'An extra dataset config to register your own datasets'
+            }
+        },
+        'dataset_param': {
+            'label': {
+                'zh': '数据集设置',
+                'en': 'Dataset settings'
+            },
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='dataset_param', open=True):
+            with gr.Row():
+                gr.Dropdown(
+                    elem_id='dataset', multiselect=True, choices=get_dataset_list(), scale=20, allow_custom_value=True)
+                gr.Textbox(elem_id='custom_dataset_info', is_list=False, scale=20)
+                gr.Slider(elem_id='split_dataset_ratio', minimum=0.0, maximum=1.0, step=0.05, scale=10)
+                gr.Slider(elem_id='max_length', minimum=32, maximum=32768, value=1024, step=1, scale=10)

ms-swift/swift/ui/llm_train/hyper.py

@@ -0,0 +1,129 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+
+
+class Hyper(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'hyper_param': {
+            'label': {
+                'zh': '超参数设置(更多参数->高级参数设置)',
+                'en': 'Hyper settings(more params->Advanced settings)',
+            },
+        },
+        'per_device_train_batch_size': {
+            'label': {
+                'zh': '训练batch size',
+                'en': 'Train batch size',
+            },
+            'info': {
+                'zh': '训练的batch size',
+                'en': 'Set the train batch size',
+            }
+        },
+        'learning_rate': {
+            'label': {
+                'zh': '学习率',
+                'en': 'Learning rate',
+            },
+            'info': {
+                'zh': '设置学习率',
+                'en': 'Set the learning rate',
+            }
+        },
+        'eval_steps': {
+            'label': {
+                'zh': '交叉验证步数',
+                'en': 'Eval steps',
+            },
+            'info': {
+                'zh': '设置每隔多少步数进行一次验证',
+                'en': 'Set the step interval to validate',
+            }
+        },
+        'num_train_epochs': {
+            'label': {
+                'zh': '数据集迭代轮次',
+                'en': 'Train epoch',
+            },
+            'info': {
+                'zh': '设置对数据集训练多少轮次',
+                'en': 'Set the max train epoch',
+            }
+        },
+        'gradient_accumulation_steps': {
+            'label': {
+                'zh': '梯度累计步数',
+                'en': 'Gradient accumulation steps',
+            },
+            'info': {
+                'zh': '设置梯度累计步数以减小显存占用',
+                'en': 'Set the gradient accumulation steps',
+            }
+        },
+        'attn_impl': {
+            'label': {
+                'zh': 'Flash Attention类型',
+                'en': 'Flash Attention Type',
+            },
+        },
+        'neftune_noise_alpha': {
+            'label': {
+                'zh': 'neftune_noise_alpha',
+                'en': 'neftune_noise_alpha'
+            },
+            'info': {
+                'zh': '使用neftune提升训练效果, 一般设置为5或者10',
+                'en': 'Use neftune to improve performance, normally the value should be 5 or 10'
+            }
+        },
+        'save_steps': {
+            'label': {
+                'zh': '存储步数',
+                'en': 'save steps',
+            },
+            'info': {
+                'zh': '设置每个多少步数进行存储',
+                'en': 'Set the save steps',
+            }
+        },
+        'output_dir': {
+            'label': {
+                'zh': '存储目录',
+                'en': 'The output dir',
+            },
+            'info': {
+                'zh': '设置输出模型存储在哪个文件夹下',
+                'en': 'Set the output folder',
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='hyper_param', open=False):
+            with gr.Blocks():
+                with gr.Row():
+                    gr.Slider(elem_id='per_device_train_batch_size', minimum=1, maximum=256, step=2, scale=20)
+                    gr.Textbox(elem_id='learning_rate', value='1e-4', lines=1, scale=20)
+                    gr.Textbox(elem_id='num_train_epochs', lines=1, scale=20)
+                    gr.Dropdown(elem_id='attn_impl', scale=20, value='flash_attn')
+                    gr.Slider(elem_id='gradient_accumulation_steps', minimum=1, maximum=256, step=2, value=16, scale=20)
+                with gr.Row():
+                    gr.Textbox(elem_id='eval_steps', lines=1, value='500', scale=20)
+                    gr.Textbox(elem_id='save_steps', value='500', lines=1, scale=20)
+                    gr.Textbox(elem_id='output_dir', scale=20)
+                    gr.Slider(elem_id='neftune_noise_alpha', minimum=0.0, maximum=20.0, step=0.5, scale=20)
+
+    @staticmethod
+    def update_lr(sft_type):
+        if sft_type == 'full':
+            return 1e-5
+        else:
+            return 1e-4

ms-swift/swift/ui/llm_train/llamapro.py

@@ -0,0 +1,40 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+
+
+class LlamaPro(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'llamapro_tab': {
+            'label': {
+                'zh': 'LLAMAPRO参数设置',
+                'en': 'LLAMAPRO Settings'
+            },
+        },
+        'llamapro_num_new_blocks': {
+            'label': {
+                'zh': 'LLAMAPRO插入层数',
+                'en': 'LLAMAPRO new layers'
+            },
+        },
+        'llamapro_num_groups': {
+            'label': {
+                'zh': 'LLAMAPRO对原模型的分组数',
+                'en': 'LLAMAPRO groups of model'
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='llamapro_tab', open=False):
+            with gr.Blocks():
+                with gr.Row():
+                    gr.Textbox(elem_id='llamapro_num_new_blocks')
+                    gr.Textbox(elem_id='llamapro_num_groups')

ms-swift/swift/ui/llm_train/llm_train.py

@@ -0,0 +1,420 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import collections
+import os
+import re
+import sys
+import time
+from functools import partial
+from subprocess import PIPE, STDOUT, Popen
+from typing import Dict, Type
+
+import gradio as gr
+import json
+import torch
+from json import JSONDecodeError
+from transformers.utils import is_torch_cuda_available, is_torch_npu_available
+
+from swift.llm import RLHFArguments
+from swift.llm.argument.base_args.base_args import get_supported_tuners
+from swift.ui.base import BaseUI
+from swift.ui.llm_train.advanced import Advanced
+from swift.ui.llm_train.dataset import Dataset
+from swift.ui.llm_train.galore import Galore
+from swift.ui.llm_train.hyper import Hyper
+from swift.ui.llm_train.lisa import Lisa
+from swift.ui.llm_train.llamapro import LlamaPro
+from swift.ui.llm_train.lora import LoRA
+from swift.ui.llm_train.model import Model
+from swift.ui.llm_train.quantization import Quantization
+from swift.ui.llm_train.report_to import ReportTo
+from swift.ui.llm_train.rlhf import RLHF
+from swift.ui.llm_train.runtime import Runtime
+from swift.ui.llm_train.save import Save
+from swift.ui.llm_train.self_cog import SelfCog
+from swift.utils import get_device_count, get_logger
+
+logger = get_logger()
+
+
+class LLMTrain(BaseUI):
+    group = 'llm_train'
+
+    sub_ui = [
+        Model,
+        Dataset,
+        Runtime,
+        Save,
+        LoRA,
+        Hyper,
+        Quantization,
+        SelfCog,
+        Advanced,
+        RLHF,
+        Lisa,
+        Galore,
+        LlamaPro,
+        ReportTo,
+    ]
+
+    locale_dict: Dict[str, Dict] = {
+        'llm_train': {
+            'label': {
+                'zh': 'LLM训练',
+                'en': 'LLM Training',
+            }
+        },
+        'train_stage': {
+            'label': {
+                'zh': '训练Stage',
+                'en': 'Train Stage'
+            },
+            'info': {
+                'zh': '请注意选择与此匹配的数据集，人类对齐配置在页面下方',
+                'en': 'Please choose matched dataset, RLHF settings is at the bottom of the page'
+            }
+        },
+        'submit_alert': {
+            'value': {
+                'zh':
+                '任务已开始，请查看tensorboard或日志记录，关闭本页面不影响训练过程',
+                'en':
+                'Task started, please check the tensorboard or log file, '
+                'closing this page does not affect training'
+            }
+        },
+        'dataset_alert': {
+            'value': {
+                'zh': '请选择或填入一个数据集',
+                'en': 'Please input or select a dataset'
+            }
+        },
+        'submit': {
+            'value': {
+                'zh': '🚀 开始训练',
+                'en': '🚀 Begin'
+            }
+        },
+        'dry_run': {
+            'label': {
+                'zh': '仅生成运行命令',
+                'en': 'Dry-run'
+            },
+            'info': {
+                'zh': '仅生成运行命令，开发者自行运行',
+                'en': 'Generate run command only, for manually running'
+            }
+        },
+        'gpu_id': {
+            'label': {
+                'zh': '选择可用GPU',
+                'en': 'Choose GPU'
+            },
+            'info': {
+                'zh': '选择训练使用的GPU号，如CUDA不可用只能选择CPU',
+                'en': 'Select GPU to train'
+            }
+        },
+        'train_type': {
+            'label': {
+                'zh': '训练方式',
+                'en': 'Train type'
+            },
+            'info': {
+                'zh': '选择训练的方式',
+                'en': 'Select the training type'
+            }
+        },
+        'seed': {
+            'label': {
+                'zh': '随机数种子',
+                'en': 'Seed'
+            },
+            'info': {
+                'zh': '选择随机数种子',
+                'en': 'Select a random seed'
+            }
+        },
+        'torch_dtype': {
+            'label': {
+                'zh': '训练精度',
+                'en': 'Training Precision'
+            },
+            'info': {
+                'zh': '选择训练精度',
+                'en': 'Select the training precision'
+            }
+        },
+        'envs': {
+            'label': {
+                'zh': '环境变量',
+                'en': 'Extra env vars'
+            },
+        },
+        'use_ddp': {
+            'label': {
+                'zh': '使用DDP',
+                'en': 'Use DDP'
+            },
+            'info': {
+                'zh': '是否使用数据并行训练',
+                'en': 'Use Distributed Data Parallel to train'
+            }
+        },
+        'ddp_num': {
+            'label': {
+                'zh': 'DDP分片数量',
+                'en': 'Number of DDP sharding'
+            },
+            'info': {
+                'zh': '启用多少进程的数据并��',
+                'en': 'The data parallel size of DDP'
+            }
+        },
+        'tuner_backend': {
+            'label': {
+                'zh': 'Tuner backend',
+                'en': 'Tuner backend'
+            },
+            'info': {
+                'zh': 'tuner实现框架',
+                'en': 'The tuner backend'
+            }
+        },
+        'use_liger_kernel': {
+            'label': {
+                'zh': '使用Liger kernel',
+                'en': 'Use Liger kernel'
+            },
+            'info': {
+                'zh': 'Liger kernel可以有效降低显存使用',
+                'en': 'Liger kernel can reduce memory usage'
+            }
+        },
+        'train_param': {
+            'label': {
+                'zh': '训练参数设置',
+                'en': 'Train settings'
+            },
+        },
+    }
+
+    choice_dict = BaseUI.get_choices_from_dataclass(RLHFArguments)
+    default_dict = BaseUI.get_default_value_from_dataclass(RLHFArguments)
+    arguments = BaseUI.get_argument_names(RLHFArguments)
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.TabItem(elem_id='llm_train', label=''):
+            default_device = 'cpu'
+            device_count = get_device_count()
+            if device_count > 0:
+                default_device = '0'
+            with gr.Blocks():
+                Model.build_ui(base_tab)
+                Dataset.build_ui(base_tab)
+                with gr.Accordion(elem_id='train_param', open=True):
+                    with gr.Row():
+                        gr.Dropdown(elem_id='train_stage', choices=['pt', 'sft', 'rlhf'], value='sft', scale=3)
+                        gr.Dropdown(elem_id='train_type', scale=2, choices=list(get_supported_tuners()))
+                        gr.Dropdown(elem_id='tuner_backend', scale=2)
+                    with gr.Row():
+                        gr.Textbox(elem_id='seed', scale=4)
+                        gr.Dropdown(elem_id='torch_dtype', scale=4)
+                        gr.Checkbox(elem_id='use_liger_kernel', scale=4)
+                        gr.Checkbox(elem_id='use_ddp', value=False, scale=4)
+                        gr.Textbox(elem_id='ddp_num', value='2', scale=4)
+                Hyper.build_ui(base_tab)
+                Runtime.build_ui(base_tab)
+                with gr.Row():
+                    gr.Dropdown(
+                        elem_id='gpu_id',
+                        multiselect=True,
+                        choices=[str(i) for i in range(device_count)] + ['cpu'],
+                        value=default_device,
+                        scale=8)
+                    gr.Textbox(elem_id='envs', scale=8)
+                    gr.Checkbox(elem_id='dry_run', value=False, scale=4)
+                    submit = gr.Button(elem_id='submit', scale=4, variant='primary')
+
+                LoRA.build_ui(base_tab)
+                RLHF.build_ui(base_tab)
+                Quantization.build_ui(base_tab)
+                Galore.build_ui(base_tab)
+                Lisa.build_ui(base_tab)
+                LlamaPro.build_ui(base_tab)
+                SelfCog.build_ui(base_tab)
+                Save.build_ui(base_tab)
+                ReportTo.build_ui(base_tab)
+                Advanced.build_ui(base_tab)
+
+                cls.element('train_type').change(
+                    Hyper.update_lr, inputs=[base_tab.element('train_type')], outputs=[cls.element('learning_rate')])
+
+                submit.click(
+                    cls.train_local,
+                    list(cls.valid_elements().values()), [
+                        cls.element('running_cmd'),
+                        cls.element('logging_dir'),
+                        cls.element('runtime_tab'),
+                        cls.element('running_tasks'),
+                        cls.element('train_record'),
+                    ],
+                    queue=True)
+
+                base_tab.element('running_tasks').change(
+                    partial(Runtime.task_changed, base_tab=base_tab), [base_tab.element('running_tasks')],
+                    list(base_tab.valid_elements().values()) + [cls.element('log')] + Runtime.all_plots)
+                Runtime.element('kill_task').click(
+                    Runtime.kill_task,
+                    [Runtime.element('running_tasks')],
+                    [Runtime.element('running_tasks')] + [Runtime.element('log')] + Runtime.all_plots,
+                ).then(Runtime.reset, [], [Runtime.element('logging_dir')] + [Hyper.element('output_dir')])
+
+    @classmethod
+    def update_runtime(cls):
+        return gr.update(open=True), gr.update(visible=True)
+
+    @classmethod
+    def train(cls, *args):
+        ignore_elements = ('logging_dir', 'more_params', 'train_stage', 'envs')
+        default_args = cls.get_default_value_from_dataclass(RLHFArguments)
+        kwargs = {}
+        kwargs_is_list = {}
+        other_kwargs = {}
+        more_params = {}
+        more_params_cmd = ''
+        keys = cls.valid_element_keys()
+        train_stage = 'sft'
+        for key, value in zip(keys, args):
+            compare_value = default_args.get(key)
+            if isinstance(value, str) and re.fullmatch(cls.int_regex, value):
+                value = int(value)
+            elif isinstance(value, str) and re.fullmatch(cls.float_regex, value):
+                value = float(value)
+            elif isinstance(value, str) and re.fullmatch(cls.bool_regex, value):
+                value = True if value.lower() == 'true' else False
+            if key not in ignore_elements and key in default_args and compare_value != value and value:
+                kwargs[key] = value if not isinstance(value, list) else ' '.join(value)
+                kwargs_is_list[key] = isinstance(value, list) or getattr(cls.element(key), 'is_list', False)
+            else:
+                other_kwargs[key] = value
+            if key == 'more_params' and value:
+                try:
+                    more_params = json.loads(value)
+                except (JSONDecodeError or TypeError):
+                    more_params_cmd = value
+
+            if key == 'train_stage':
+                train_stage = value
+
+        kwargs.update(more_params)
+        if 'dataset' not in kwargs and 'custom_train_dataset_path' not in kwargs:
+            raise gr.Error(cls.locale('dataset_alert', cls.lang)['value'])
+
+        model = kwargs.get('model')
+        if os.path.exists(model) and os.path.exists(os.path.join(model, 'args.json')):
+            kwargs['resume_from_checkpoint'] = kwargs.pop('model')
+
+        cmd = train_stage
+        if kwargs.get('deepspeed'):
+            more_params_cmd += f' --deepspeed {kwargs.pop("deepspeed")} '
+        try:
+            sft_args = RLHFArguments(
+                **{
+                    key: value.split(' ') if kwargs_is_list.get(key, False) and isinstance(value, str) else value
+                    for key, value in kwargs.items()
+                })
+        except Exception as e:
+            if 'using `--model`' in str(e):  # TODO a dirty fix
+                kwargs['model'] = kwargs.pop('resume_from_checkpoint')
+                sft_args = RLHFArguments(
+                    **{
+                        key: value.split(' ') if kwargs_is_list.get(key, False) and isinstance(value, str) else value
+                        for key, value in kwargs.items()
+                    })
+            else:
+                raise e
+        params = ''
+
+        sep = f'{cls.quote} {cls.quote}'
+        for e in kwargs:
+            if isinstance(kwargs[e], list):
+                params += f'--{e} {cls.quote}{sep.join(kwargs[e])}{cls.quote} '
+            elif e in kwargs_is_list and kwargs_is_list[e]:
+                all_args = [arg for arg in kwargs[e].split(' ') if arg.strip()]
+                params += f'--{e} {cls.quote}{sep.join(all_args)}{cls.quote} '
+            else:
+                params += f'--{e} {cls.quote}{kwargs[e]}{cls.quote} '
+        params += more_params_cmd + ' '
+        params += f'--add_version False --output_dir {sft_args.output_dir} ' \
+                  f'--logging_dir {sft_args.logging_dir} --ignore_args_error True'
+        ddp_param = ''
+        devices = other_kwargs['gpu_id']
+        envs = other_kwargs['envs'] or ''
+        envs = envs.strip()
+        devices = [d for d in devices if d]
+        if other_kwargs['use_ddp']:
+            assert int(other_kwargs['ddp_num']) > 0
+            ddp_param = f'NPROC_PER_NODE={int(other_kwargs["ddp_num"])}'
+        assert (len(devices) == 1 or 'cpu' not in devices)
+        gpus = ','.join(devices)
+        cuda_param = ''
+        if gpus != 'cpu':
+            if is_torch_npu_available():
+                cuda_param = f'ASCEND_RT_VISIBLE_DEVICES={gpus}'
+            elif is_torch_cuda_available():
+                cuda_param = f'CUDA_VISIBLE_DEVICES={gpus}'
+            else:
+                cuda_param = ''
+
+        log_file = os.path.join(sft_args.logging_dir, 'run.log')
+        if sys.platform == 'win32':
+            if cuda_param:
+                cuda_param = f'set {cuda_param} && '
+            if ddp_param:
+                ddp_param = f'set {ddp_param} && '
+            if envs:
+                envs = [env.strip() for env in envs.split(' ') if env.strip()]
+                _envs = ''
+                for env in envs:
+                    _envs += f'set {env} && '
+                envs = _envs
+            run_command = f'{cuda_param}{ddp_param}{envs}start /b swift sft {params} > {log_file} 2>&1'
+        else:
+            run_command = f'{cuda_param} {ddp_param} {envs} nohup swift {cmd} {params} > {log_file} 2>&1 &'
+        logger.info(f'Run training: {run_command}')
+        if model:
+            record = {}
+            for key, value in zip(keys, args):
+                if key in default_args or key in ('more_params', 'train_stage', 'use_ddp', 'ddp_num', 'gpu_id', 'envs'):
+                    record[key] = value or None
+            cls.save_cache(model, record)
+        return run_command, sft_args, other_kwargs
+
+    @classmethod
+    def train_studio(cls, *args):
+        run_command, sft_args, other_kwargs = cls.train(*args)
+        if not other_kwargs['dry_run']:
+            lines = collections.deque(maxlen=int(os.environ.get('MAX_LOG_LINES', 50)))
+            process = Popen(run_command, shell=True, stdout=PIPE, stderr=STDOUT)
+            with process.stdout:
+                for line in iter(process.stdout.readline, b''):
+                    line = line.decode('utf-8')
+                    lines.append(line)
+                    yield ['\n'.join(lines)] + Runtime.plot(run_command) + [run_command]
+        else:
+            yield [
+                'Current is dryrun mode so you can only view the training cmd, please duplicate this space to '
+                'do training or use with inference.'
+            ] + [None] * len(Runtime.sft_plot) + [run_command]
+
+    @classmethod
+    def train_local(cls, *args):
+        run_command, sft_args, other_kwargs = cls.train(*args)
+        if not other_kwargs['dry_run']:
+            os.makedirs(sft_args.logging_dir, exist_ok=True)
+            os.system(run_command)
+            time.sleep(1)  # to make sure the log file has been created.
+            gr.Info(cls.locale('submit_alert', cls.lang)['value'])
+        return run_command, sft_args.logging_dir, gr.update(open=True), Runtime.refresh_tasks(
+            sft_args.output_dir), gr.update(choices=cls.list_cache(sft_args.model))

ms-swift/swift/ui/llm_train/lora.py

@@ -0,0 +1,102 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+
+
+class LoRA(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'lora_tab': {
+            'label': {
+                'zh': 'LoRA参数设置',
+                'en': 'LoRA settings'
+            },
+        },
+        'target_modules': {
+            'label': {
+                'zh': 'LoRA目标模块',
+                'en': 'LoRA target modules'
+            },
+            'info': {
+                'zh': '设置LoRA目标模块，如训练所有Linear请改为`all-linear`',
+                'en': 'Set the LoRA target modules, fill in `all-linear` if train all Linears'
+            }
+        },
+        'lora_rank': {
+            'label': {
+                'zh': 'LoRA的秩',
+                'en': 'The LoRA rank'
+            }
+        },
+        'lora_alpha': {
+            'label': {
+                'zh': 'LoRA的alpha',
+                'en': 'The LoRA alpha'
+            }
+        },
+        'lora_dropout': {
+            'label': {
+                'zh': 'LoRA的dropout',
+                'en': 'The LoRA dropout'
+            }
+        },
+        'use_rslora': {
+            'label': {
+                'zh': '使用rslora',
+                'en': 'Use rslora'
+            }
+        },
+        'use_dora': {
+            'label': {
+                'zh': '使用dora',
+                'en': 'Use dora'
+            }
+        },
+        'lora_dtype': {
+            'label': {
+                'zh': 'lora部分的参数类型',
+                'en': 'The dtype of lora parameters'
+            }
+        },
+        'init_weights': {
+            'label': {
+                'zh': 'lora初始化方法',
+                'en': 'init lora weights'
+            },
+            'info': {
+                'zh': 'gaussian/pissa/pissa_niter_[n]/olora/loftq/true/false',
+                'en': 'gaussian/pissa/pissa_niter_[n]/olora/loftq/true/false',
+            }
+        },
+        'lorap_lr_ratio': {
+            'label': {
+                'zh': 'Lora+学习率倍率',
+                'en': 'The lr ratio of Lora+'
+            },
+            'info': {
+                'zh': '建议值16.0',
+                'en': 'Suggested value: 16.0'
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='lora_tab', open=True):
+            with gr.Blocks():
+                with gr.Row():
+                    gr.Textbox(elem_id='target_modules', lines=1, scale=5, value='all-linear', is_list=True)
+                    gr.Slider(elem_id='lora_rank', value=8, minimum=1, maximum=512, step=8, scale=2)
+                    gr.Slider(elem_id='lora_alpha', value=32, minimum=1, maximum=512, step=8, scale=2)
+                    gr.Textbox(elem_id='lora_dropout', scale=2)
+                with gr.Row():
+                    gr.Dropdown(elem_id='lora_dtype', scale=2, value=None)
+                    gr.Textbox(elem_id='lorap_lr_ratio', scale=2)
+                    gr.Checkbox(elem_id='use_rslora', scale=2)
+                    gr.Checkbox(elem_id='use_dora', scale=2)
+                    gr.Textbox(elem_id='init_weights', scale=4)

ms-swift/swift/ui/llm_train/model.py

@@ -0,0 +1,127 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from functools import partial
+from typing import Type
+
+import gradio as gr
+
+from swift.llm import TEMPLATE_MAPPING, ModelType, RLHFArguments
+from swift.llm.model.register import get_all_models
+from swift.ui.base import BaseUI
+
+
+class Model(BaseUI):
+    group = 'llm_train'
+
+    locale_dict = {
+        'model_type': {
+            'label': {
+                'zh': '模型类型',
+                'en': 'Select Model Type'
+            },
+            'info': {
+                'zh': 'SWIFT已支持的模型类型',
+                'en': 'Base model type supported by SWIFT'
+            }
+        },
+        'model': {
+            'label': {
+                'zh': '模型id或路径',
+                'en': 'Model id or path'
+            },
+            'info': {
+                'zh': '实际的模型id',
+                'en': 'The actual model id or model path'
+            }
+        },
+        'template': {
+            'label': {
+                'zh': '模型Prompt模板类型',
+                'en': 'Prompt template type'
+            },
+            'info': {
+                'zh': '选择匹配模型的Prompt模板',
+                'en': 'Choose the template type of the model'
+            }
+        },
+        'system': {
+            'label': {
+                'zh': 'system字段',
+                'en': 'system'
+            },
+            'info': {
+                'zh': '选择system字段的内容',
+                'en': 'Choose the content of the system field'
+            }
+        },
+        'reset': {
+            'value': {
+                'zh': '恢复模型初始值',
+                'en': 'Reset model default'
+            },
+        },
+        'train_record': {
+            'label': {
+                'zh': '训练记录',
+                'en': 'Train record'
+            },
+            'info': {
+                'zh': '展示使用web-ui的历史训练及参数',
+                'en': 'Show the training history and parameters'
+            }
+        },
+        'clear_cache': {
+            'value': {
+                'zh': '删除训练记录',
+                'en': 'Delete train records'
+            },
+        },
+        'model_param': {
+            'label': {
+                'zh': '模型设置',
+                'en': 'Model settings'
+            },
+        },
+        'checkpoint': {
+            'value': {
+                'zh': '训练后的模型',
+                'en': 'Trained model'
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='model_param', open=True):
+            with gr.Row():
+                model = gr.Dropdown(
+                    elem_id='model',
+                    scale=20,
+                    choices=get_all_models(),
+                    value='Qwen/Qwen2.5-7B-Instruct',
+                    allow_custom_value=True)
+                gr.Dropdown(elem_id='model_type', choices=ModelType.get_model_name_list(), scale=20)
+                gr.Dropdown(elem_id='template', choices=list(TEMPLATE_MAPPING.keys()), scale=20)
+                train_record = gr.Dropdown(elem_id='train_record', choices=[], scale=20)
+                clear_cache = gr.Button(elem_id='clear_cache', scale=2)
+            with gr.Row():
+                gr.Textbox(elem_id='system', lines=1, scale=20)
+
+        def clear_record(model):
+            if model:
+                cls.clear_cache(model)
+                return gr.update(choices=[])
+            return gr.update()
+
+        clear_cache.click(clear_record, inputs=[model], outputs=[train_record])
+
+    @classmethod
+    def after_build_ui(cls, base_tab: Type['BaseUI']):
+        cls.element('model').change(
+            partial(base_tab.update_input_model, arg_cls=RLHFArguments),
+            inputs=[cls.element('model')],
+            outputs=[cls.element('train_record')] + list(base_tab.valid_elements().values()))
+
+        cls.element('train_record').change(
+            partial(base_tab.update_all_settings, base_tab=base_tab),
+            inputs=[cls.element('model'), cls.element('train_record')],
+            outputs=list(base_tab.valid_elements().values()))

ms-swift/swift/ui/llm_train/quantization.py

@@ -0,0 +1,68 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+
+
+class Quantization(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'quantization_tab': {
+            'label': {
+                'zh': '量化参数设置',
+                'en': 'Quantization settings'
+            },
+        },
+        'quant_method': {
+            'label': {
+                'zh': '量化方式',
+                'en': 'Quantization method'
+            },
+            'info': {
+                'zh': '如果制定了量化位数，本参数默认为bnb',
+                'en': 'Default is bnb if quantization_bit is specified'
+            }
+        },
+        'quant_bits': {
+            'label': {
+                'zh': '量化bit数',
+                'en': 'Quantization bit'
+            },
+            'info': {
+                'zh': '设置量化bit数, 0代表不进行量化',
+                'en': 'Set the quantization bit, 0 for no quantization'
+            }
+        },
+        'bnb_4bit_compute_dtype': {
+            'label': {
+                'zh': 'bnb_4bit_compute_dtype',
+                'en': 'bnb_4bit_compute_dtype'
+            },
+        },
+        'bnb_4bit_quant_type': {
+            'label': {
+                'zh': 'bnb_4bit_quant_type',
+                'en': 'bnb_4bit_quant_type'
+            },
+        },
+        'bnb_4bit_use_double_quant': {
+            'label': {
+                'zh': 'bnb_4bit_use_double_quant',
+                'en': 'bnb_4bit_use_double_quant'
+            },
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='quantization_tab', open=False):
+            with gr.Row():
+                gr.Dropdown(elem_id='quant_bits', value=None)
+                gr.Dropdown(elem_id='quant_method', value=None)
+                gr.Dropdown(elem_id='bnb_4bit_compute_dtype', value=None)
+                gr.Dropdown(elem_id='bnb_4bit_quant_type', value=None)
+                gr.Checkbox(elem_id='bnb_4bit_use_double_quant', value=None)

ms-swift/swift/ui/llm_train/report_to.py

@@ -0,0 +1,75 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+
+
+class ReportTo(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'reporter': {
+            'label': {
+                'zh': '训练记录',
+                'en': 'Training report'
+            },
+        },
+        'report_to': {
+            'label': {
+                'zh': '训练记录方式',
+                'en': 'Report to'
+            },
+        },
+        'swanlab_token': {
+            'label': {
+                'zh': 'swanlab登录token',
+                'en': 'The login token of swanlab'
+            },
+        },
+        'swanlab_project': {
+            'label': {
+                'zh': 'swanlab项目名称',
+                'en': 'Project of swanlab'
+            },
+        },
+        'swanlab_workspace': {
+            'label': {
+                'zh': 'swanlab工作空间',
+                'en': 'Workspace of swanlab'
+            },
+        },
+        'swanlab_exp_name': {
+            'label': {
+                'zh': 'swanlab实验名称',
+                'en': 'Experiment of swanlab'
+            },
+        },
+        'swanlab_mode': {
+            'label': {
+                'zh': 'swanlab工作模式',
+                'en': 'Work mode of swanlab'
+            },
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='reporter', open=False):
+            with gr.Blocks():
+                with gr.Row():
+                    gr.Dropdown(
+                        elem_id='report_to',
+                        multiselect=True,
+                        is_list=True,
+                        choices=['tensorboard', 'wandb', 'swanlab'],
+                        allow_custom_value=True,
+                        scale=20)
+                    gr.Textbox(elem_id='swanlab_token', lines=1, scale=20)
+                    gr.Textbox(elem_id='swanlab_project', lines=1, scale=20)
+                with gr.Row():
+                    gr.Textbox(elem_id='swanlab_workspace', lines=1, scale=20)
+                    gr.Textbox(elem_id='swanlab_exp_name', lines=1, scale=20)
+                    gr.Dropdown(elem_id='swanlab_mode', scale=20)

ms-swift/swift/ui/llm_train/rlhf.py

@@ -0,0 +1,102 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from functools import partial
+from typing import Type
+
+import gradio as gr
+
+from swift.llm import ModelType
+from swift.llm.model.register import get_all_models
+from swift.ui.base import BaseUI
+
+
+class RLHF(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'rlhf_tab': {
+            'label': {
+                'zh': '人类对齐参数设置',
+                'en': 'RLHF settings'
+            },
+        },
+        'rlhf_type': {
+            'label': {
+                'zh': '人类对齐算法类型',
+                'en': 'RLHF type'
+            },
+        },
+        'ref_model_type': {
+            'label': {
+                'zh': '选择ref模型',
+                'en': 'Select ref model'
+            },
+            'info': {
+                'zh': 'SWIFT已支持的模型名称',
+                'en': 'Base model supported by SWIFT'
+            }
+        },
+        'ref_model': {
+            'label': {
+                'zh': 'ref模型id或路径',
+                'en': 'Ref model id or path'
+            },
+            'info': {
+                'zh': '实际的模型id或路径',
+                'en': 'The actual model id or path'
+            }
+        },
+        'beta': {
+            'label': {
+                'zh': 'KL正则项系数',
+                'en': 'KL regression ratio'
+            },
+        },
+        'rpo_alpha': {
+            'label': {
+                'zh': 'DPO中混合sft交叉熵的系数',
+                'en': 'DPO Cross Entropy ratio'
+            },
+        },
+        'simpo_gamma': {
+            'label': {
+                'zh': 'SimPO reward margin',
+                'en': 'SimPO reward margin'
+            },
+        },
+        'desirable_weight': {
+            'label': {
+                'zh': 'KTO符合项系数',
+                'en': 'KTO desirable ratio'
+            },
+        },
+        'undesirable_weight': {
+            'label': {
+                'zh': 'KTO不符合项系数',
+                'en': 'KTO undesirable ratio'
+            },
+        }
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='rlhf_tab', open=False):
+            with gr.Blocks():
+                with gr.Row():
+                    gr.Dropdown(elem_id='rlhf_type', value=None)
+                    gr.Dropdown(
+                        elem_id='ref_model', scale=20, value=None, choices=get_all_models(), allow_custom_value=True)
+                    gr.Dropdown(elem_id='ref_model_type', choices=ModelType.get_model_name_list(), value=None, scale=20)
+                with gr.Row():
+                    gr.Slider(elem_id='beta', minimum=0., maximum=5.0, step=0.1, scale=20)
+                    gr.Slider(elem_id='rpo_alpha', minimum=0., maximum=2, step=0.1, scale=20)
+                    gr.Slider(elem_id='simpo_gamma', minimum=0., maximum=2.0, step=0.1, scale=20)
+                    gr.Slider(elem_id='desirable_weight', minimum=0., maximum=2.0, step=0.1, scale=20)
+                    gr.Slider(elem_id='undesirable_weight', minimum=0., maximum=2.0, step=0.1, scale=20)
+
+    @classmethod
+    def after_build_ui(cls, base_tab: Type['BaseUI']):
+        cls.element('ref_model').change(
+            partial(cls.update_input_model, allow_keys=['ref_model_type'], has_record=False, is_ref_model=True),
+            inputs=[cls.element('ref_model')],
+            outputs=[cls.element('ref_model_type')])

ms-swift/swift/ui/llm_train/runtime.py

@@ -0,0 +1,571 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import collections
+import os.path
+import sys
+import time
+import webbrowser
+from datetime import datetime
+from typing import Dict, List, Tuple, Type
+
+import gradio as gr
+import json
+import matplotlib.pyplot as plt
+import psutil
+from packaging import version
+from transformers import is_tensorboard_available
+
+from swift.ui.base import BaseUI
+from swift.ui.llm_train.utils import close_loop, run_command_in_subprocess
+from swift.utils import TB_COLOR, TB_COLOR_SMOOTH, get_logger, read_tensorboard_file, tensorboard_smoothing
+from swift.utils.utils import format_time
+
+logger = get_logger()
+
+
+class Runtime(BaseUI):
+
+    handlers: Dict[str, Tuple[List, Tuple]] = {}
+
+    group = 'llm_train'
+
+    all_plots = None
+
+    log_event = {}
+
+    sft_plot = [
+        {
+            'name': 'train/loss',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'train/acc',
+            'smooth': None,
+        },
+        {
+            'name': 'train/learning_rate',
+            'smooth': None,
+        },
+        {
+            'name': 'eval/loss',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'eval/acc',
+            'smooth': None,
+        },
+    ]
+
+    dpo_plot = [
+        {
+            'name': 'train/loss',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'train/rewards/accuracies',
+            'smooth': None,
+        },
+        {
+            'name': 'train/rewards/margins',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'train/logps/chosen',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'train/logps/rejected',
+            'smooth': 0.9,
+        },
+    ]
+
+    kto_plot = [
+        {
+            'name': 'kl',
+            'smooth': None,
+        },
+        {
+            'name': 'rewards/chosen_sum',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'logps/chosen_sum',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'rewards/rejected_sum',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'logps/rejected_sum',
+            'smooth': 0.9,
+        },
+    ]
+
+    orpo_plot = [
+        {
+            'name': 'train/loss',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'train/rewards/accuracies',
+            'smooth': None,
+        },
+        {
+            'name': 'train/rewards/margins',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'train/rewards/chosen',
+            'smooth': 0.9,
+        },
+        {
+            'name': 'train/log_odds_ratio',
+            'smooth': 0.9,
+        },
+    ]
+
+    locale_dict = {
+        'runtime_tab': {
+            'label': {
+                'zh': '运行时',
+                'en': 'Runtime'
+            },
+        },
+        'tb_not_found': {
+            'value': {
+                'zh': 'tensorboard未安装,使用pip install tensorboard进行安装',
+                'en': 'tensorboard not found, install it by pip install tensorboard',
+            }
+        },
+        'running_cmd': {
+            'label': {
+                'zh': '运行命令',
+                'en': 'Command line'
+            },
+            'info': {
+                'zh': '执行的实际命令',
+                'en': 'The actual command'
+            }
+        },
+        'show_log': {
+            'value': {
+                'zh': '展示运行状态',
+                'en': 'Show running status'
+            },
+        },
+        'stop_show_log': {
+            'value': {
+                'zh': '停止展示运行状态',
+                'en': 'Stop showing running status'
+            },
+        },
+        'logging_dir': {
+            'label': {
+                'zh': '日志路径',
+                'en': 'Logging dir'
+            },
+            'info': {
+                'zh': '支持手动传入文件路径',
+                'en': 'Support fill custom path in'
+            }
+        },
+        'log': {
+            'label': {
+                'zh': '日志输出',
+                'en': 'Logging content'
+            },
+            'info': {
+                'zh': '如果日志无更新请再次点击"展示日志内容"',
+                'en': 'Please press "Show log" if the log content is not updating'
+            }
+        },
+        'running_tasks': {
+            'label': {
+                'zh': '运行中任务',
+                'en': 'Running Tasks'
+            },
+            'info': {
+                'zh': '运行中的任务（所有的swift sft命令）',
+                'en': 'All running tasks(started by swift sft)'
+            }
+        },
+        'refresh_tasks': {
+            'value': {
+                'zh': '找回运行时任务',
+                'en': 'Find running tasks'
+            },
+        },
+        'kill_task': {
+            'value': {
+                'zh': '杀死任务',
+                'en': 'Kill running task'
+            },
+        },
+        'tb_url': {
+            'label': {
+                'zh': 'Tensorboard链接',
+                'en': 'Tensorboard URL'
+            },
+            'info': {
+                'zh': '仅展示，不可编辑',
+                'en': 'Not editable'
+            }
+        },
+        'start_tb': {
+            'value': {
+                'zh': '打开TensorBoard',
+                'en': 'Start TensorBoard'
+            },
+        },
+        'close_tb': {
+            'value': {
+                'zh': '关闭TensorBoard',
+                'en': 'Close TensorBoard'
+            },
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='runtime_tab', open=False, visible=True):
+            with gr.Blocks():
+                with gr.Row():
+                    gr.Textbox(elem_id='running_cmd', lines=1, scale=20, interactive=False, max_lines=1)
+                    gr.Textbox(elem_id='logging_dir', lines=1, scale=20, max_lines=1)
+                    gr.Button(elem_id='show_log', scale=2, variant='primary')
+                    gr.Button(elem_id='stop_show_log', scale=2)
+                    gr.Textbox(elem_id='tb_url', lines=1, scale=10, interactive=False, max_lines=1)
+                    gr.Button(elem_id='start_tb', scale=2, variant='primary')
+                    gr.Button(elem_id='close_tb', scale=2)
+                with gr.Row():
+                    gr.Textbox(elem_id='log', lines=6, visible=False)
+                with gr.Row():
+                    gr.Dropdown(elem_id='running_tasks', scale=10)
+                    gr.Button(elem_id='refresh_tasks', scale=1)
+                    gr.Button(elem_id='kill_task', scale=1)
+
+                with gr.Row():
+                    cls.all_plots = []
+                    for idx, k in enumerate(Runtime.sft_plot):
+                        name = k['name']
+                        cls.all_plots.append(gr.Plot(elem_id=str(idx), label=name))
+
+                concurrency_limit = {}
+                if version.parse(gr.__version__) >= version.parse('4.0.0'):
+                    concurrency_limit = {'concurrency_limit': 5}
+                base_tab.element('show_log').click(
+                    Runtime.update_log, [base_tab.element('running_tasks')], [cls.element('log')] + cls.all_plots).then(
+                        Runtime.wait, [base_tab.element('logging_dir'),
+                                       base_tab.element('running_tasks')], [cls.element('log')] + cls.all_plots,
+                        **concurrency_limit)
+
+                base_tab.element('stop_show_log').click(cls.break_log_event, [cls.element('running_tasks')], [])
+
+                base_tab.element('start_tb').click(
+                    Runtime.start_tb,
+                    [base_tab.element('logging_dir')],
+                    [base_tab.element('tb_url')],
+                )
+
+                base_tab.element('close_tb').click(
+                    Runtime.close_tb,
+                    [base_tab.element('logging_dir')],
+                    [],
+                )
+
+                base_tab.element('refresh_tasks').click(
+                    Runtime.refresh_tasks,
+                    [base_tab.element('running_tasks')],
+                    [base_tab.element('running_tasks')],
+                )
+
+    @classmethod
+    def get_plot(cls, task):
+        if not task or 'swift sft' in task or 'swift pt' in task:
+            return cls.sft_plot
+
+        args: dict = cls.parse_info_from_cmdline(task)[1]
+        train_type = args.get('rlhf_type', 'dpo')
+        if train_type in ('dpo', 'cpo', 'simpo'):
+            return cls.dpo_plot
+        elif train_type == 'kto':
+            return cls.kto_plot
+        elif train_type == 'orpo':
+            return cls.orpo_plot
+
+    @classmethod
+    def update_log(cls, task):
+        ret = [gr.update(visible=True)]
+        plot = Runtime.get_plot(task)
+        for i in range(len(plot)):
+            p = plot[i]
+            ret.append(gr.update(visible=True, label=p['name']))
+        return ret
+
+    @classmethod
+    def get_initial(cls, line):
+        tqdm_starts = ['Train:', 'Map:', 'Val:', 'Filter:']
+        for start in tqdm_starts:
+            if line.startswith(start):
+                return start
+        return None
+
+    @classmethod
+    def wait(cls, logging_dir, task):
+        if not logging_dir:
+            return [None] + Runtime.plot(task)
+        log_file = os.path.join(logging_dir, 'run.log')
+        cls.log_event[logging_dir] = False
+        offset = 0
+        latest_data = ''
+        lines = collections.deque(maxlen=int(os.environ.get('MAX_LOG_LINES', 50)))
+        try:
+            with open(log_file, 'r', encoding='utf-8') as input:
+                input.seek(offset)
+                fail_cnt = 0
+                while True:
+                    try:
+                        latest_data += input.read()
+                    except UnicodeDecodeError:
+                        continue
+                    if not latest_data:
+                        time.sleep(0.5)
+                        fail_cnt += 1
+                        if fail_cnt > 50:
+                            break
+
+                    if cls.log_event.get(logging_dir, False):
+                        cls.log_event[logging_dir] = False
+                        break
+
+                    if '\n' not in latest_data:
+                        continue
+                    latest_lines = latest_data.split('\n')
+                    if latest_data[-1] != '\n':
+                        latest_data = latest_lines[-1]
+                        latest_lines = latest_lines[:-1]
+                    else:
+                        latest_data = ''
+                    lines.extend(latest_lines)
+                    start = cls.get_initial(lines[-1])
+                    if start:
+                        i = len(lines) - 2
+                        while i >= 0:
+                            if lines[i].startswith(start):
+                                del lines[i]
+                                i -= 1
+                            else:
+                                break
+                    yield ['\n'.join(lines)] + Runtime.plot(task)
+        except IOError:
+            pass
+
+    @classmethod
+    def break_log_event(cls, task):
+        if not task:
+            return
+        pid, all_args = Runtime.parse_info_from_cmdline(task)
+        cls.log_event[all_args['logging_dir']] = True
+
+    @classmethod
+    def show_log(cls, logging_dir):
+        webbrowser.open('file://' + os.path.join(logging_dir, 'run.log'), new=2)
+
+    @classmethod
+    def start_tb(cls, logging_dir):
+        if not is_tensorboard_available():
+            gr.Error(cls.locale('tb_not_found', cls.lang)['value'])
+            return ''
+
+        logging_dir = logging_dir.strip()
+        logging_dir = logging_dir if not logging_dir.endswith(os.sep) else logging_dir[:-1]
+        if logging_dir in cls.handlers:
+            return cls.handlers[logging_dir][1]
+
+        handler, lines = run_command_in_subprocess('tensorboard', '--logdir', logging_dir, timeout=2)
+        localhost_addr = ''
+        for line in lines:
+            if 'http://localhost:' in line:
+                line = line[line.index('http://localhost:'):]
+                localhost_addr = line[:line.index(' ')]
+        cls.handlers[logging_dir] = (handler, localhost_addr)
+        logger.info('===========Tensorboard Log============')
+        logger.info('\n'.join(lines))
+        webbrowser.open(localhost_addr, new=2)
+        return localhost_addr
+
+    @staticmethod
+    def close_tb(logging_dir):
+        if logging_dir in Runtime.handlers:
+            close_loop(Runtime.handlers[logging_dir][0])
+            Runtime.handlers.pop(logging_dir)
+
+    @staticmethod
+    def refresh_tasks(running_task=None):
+        output_dir = running_task if not running_task or 'pid:' not in running_task else None
+        process_name = 'swift'
+        negative_name = 'swift.exe'
+        cmd_name = ['pt', 'sft', 'rlhf']
+        process = []
+        selected = None
+        for proc in psutil.process_iter():
+            try:
+                cmdlines = proc.cmdline()
+            except (psutil.ZombieProcess, psutil.AccessDenied, psutil.NoSuchProcess):
+                cmdlines = []
+            if any([process_name in cmdline
+                    for cmdline in cmdlines]) and not any([negative_name in cmdline
+                                                           for cmdline in cmdlines]) and any(  # noqa
+                                                               [cmdline in cmd_name for cmdline in cmdlines]):  # noqa
+                process.append(Runtime.construct_running_task(proc))
+                if output_dir is not None and any(  # noqa
+                    [output_dir == cmdline for cmdline in cmdlines]):  # noqa
+                    selected = Runtime.construct_running_task(proc)
+        if not selected:
+            if running_task and running_task in process:
+                selected = running_task
+        if not selected and process:
+            selected = process[0]
+        return gr.update(choices=process, value=selected)
+
+    @staticmethod
+    def construct_running_task(proc):
+        pid = proc.pid
+        ts = time.time()
+        create_time = proc.create_time()
+        create_time_formatted = datetime.fromtimestamp(create_time).strftime('%Y-%m-%d, %H:%M')
+
+        return f'pid:{pid}/create:{create_time_formatted}' \
+               f'/running:{format_time(ts-create_time)}/cmd:{" ".join(proc.cmdline())}'
+
+    @staticmethod
+    def parse_info_from_cmdline(task):
+        pid = None
+        if '/cmd:' in task:
+            for i in range(3):
+                slash = task.find('/')
+                if i == 0:
+                    pid = task[:slash].split(':')[1]
+                task = task[slash + 1:]
+        if 'swift sft' in task:
+            args = task.split('swift sft')[1]
+        elif 'swift pt' in task:
+            args = task.split('swift pt')[1]
+        elif 'swift rlhf' in task:
+            args = task.split('swift rlhf')[1]
+        else:
+            raise ValueError(f'Cannot parse cmd line: {task}')
+        args = [arg.strip() for arg in args.split('--') if arg.strip()]
+        all_args = {}
+        for i in range(len(args)):
+            space = args[i].find(' ')
+            splits = args[i][:space], args[i][space + 1:]
+            all_args[splits[0]] = splits[1]
+
+        output_dir = all_args['output_dir']
+        if os.path.exists(os.path.join(output_dir, 'args.json')):
+            with open(os.path.join(output_dir, 'args.json'), 'r', encoding='utf-8') as f:
+                _json = json.load(f)
+            for key in all_args.keys():
+                all_args[key] = _json.get(key)
+                if isinstance(all_args[key], list):
+                    if any([' ' in value for value in all_args[key]]):
+                        all_args[key] = [f'"{value}"' for value in all_args[key]]
+                    all_args[key] = ' '.join(all_args[key])
+        return pid, all_args
+
+    @staticmethod
+    def kill_task(task):
+        if task:
+            pid, all_args = Runtime.parse_info_from_cmdline(task)
+            output_dir = all_args['output_dir']
+            if sys.platform == 'win32':
+                os.system(f'taskkill /f /t /pid "{pid}"')
+            else:
+                os.system(f'pkill -9 -f {output_dir}')
+            time.sleep(1)
+            Runtime.break_log_event(task)
+        return [Runtime.refresh_tasks()] + [gr.update(value=None)] * (len(Runtime.get_plot(task)) + 1)
+
+    @staticmethod
+    def reset():
+        return None, 'output'
+
+    @staticmethod
+    def task_changed(task, base_tab):
+        if task:
+            _, all_args = Runtime.parse_info_from_cmdline(task)
+        else:
+            all_args = {}
+        elements = list(base_tab.valid_elements().values())
+        ret = []
+        for e in elements:
+            if e.elem_id in all_args:
+                if isinstance(e, gr.Dropdown) and e.multiselect:
+                    arg = all_args[e.elem_id].split(' ')
+                else:
+                    arg = all_args[e.elem_id]
+                ret.append(gr.update(value=arg))
+            else:
+                ret.append(gr.update())
+        Runtime.break_log_event(task)
+        return ret + [gr.update(value=None)] * (len(Runtime.get_plot(task)) + 1)
+
+    @staticmethod
+    def plot(task):
+        plot = Runtime.get_plot(task)
+        if not task:
+            return [None] * len(plot)
+        _, all_args = Runtime.parse_info_from_cmdline(task)
+        tb_dir = all_args['logging_dir']
+        if not os.path.exists(tb_dir):
+            return [None] * len(plot)
+        fname = [
+            fname for fname in os.listdir(tb_dir)
+            if os.path.isfile(os.path.join(tb_dir, fname)) and fname.startswith('events.out')
+        ]
+        if fname:
+            fname = fname[0]
+        else:
+            return [None] * len(plot)
+        tb_path = os.path.join(tb_dir, fname)
+        data = read_tensorboard_file(tb_path)
+
+        plots = []
+        for k in plot:
+            name = k['name']
+            smooth = k['smooth']
+            if name == 'train/acc':
+                if 'train/token_acc' in data:
+                    name = 'train/token_acc'
+                if 'train/seq_acc' in data:
+                    name = 'train/seq_acc'
+            if name == 'eval/acc':
+                if 'eval/token_acc' in data:
+                    name = 'eval/token_acc'
+                if 'eval/seq_acc' in data:
+                    name = 'eval/seq_acc'
+            if name not in data:
+                plots.append(None)
+                continue
+            _data = data[name]
+            steps = [d['step'] for d in _data]
+            values = [d['value'] for d in _data]
+            if len(values) == 0:
+                continue
+
+            plt.close('all')
+            fig = plt.figure()
+            ax = fig.add_subplot()
+            # _, ax = plt.subplots(1, 1, squeeze=True, figsize=(8, 5), dpi=100)
+            ax.set_title(name)
+            if len(values) == 1:
+                ax.scatter(steps, values, color=TB_COLOR_SMOOTH)
+            elif smooth is not None:
+                ax.plot(steps, values, color=TB_COLOR)
+                values_s = tensorboard_smoothing(values, smooth)
+                ax.plot(steps, values_s, color=TB_COLOR_SMOOTH)
+            else:
+                ax.plot(steps, values, color=TB_COLOR_SMOOTH)
+            plots.append(fig)
+        return plots

ms-swift/swift/ui/llm_train/save.py

@@ -0,0 +1,84 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+
+
+class Save(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'save_param': {
+            'label': {
+                'zh': '存储参数设置',
+                'en': 'Saving settings'
+            },
+        },
+        'push_to_hub': {
+            'label': {
+                'zh': '推送魔搭Hub',
+                'en': 'Push to modelscope hub',
+            },
+            'info': {
+                'zh': '是否推送魔搭的模型库',
+                'en': 'Whether push the output model to modelscope hub',
+            }
+        },
+        'hub_model_id': {
+            'label': {
+                'zh': '魔搭模型id',
+                'en': 'The model-id in modelscope',
+            },
+            'info': {
+                'zh': '设置魔搭的模型id',
+                'en': 'Set the model-id of modelscope',
+            }
+        },
+        'hub_private_repo': {
+            'label': {
+                'zh': '设置仓库私有',
+                'en': 'Model is private',
+            },
+            'info': {
+                'zh': '以私有方式推送魔搭hub',
+                'en': 'Set the model as private',
+            }
+        },
+        'hub_strategy': {
+            'label': {
+                'zh': '推送策略',
+                'en': 'Push strategy',
+            },
+            'info': {
+                'zh': '设置模型推送策略',
+                'en': 'Set the push strategy',
+            }
+        },
+        'hub_token': {
+            'label': {
+                'zh': '仓库token',
+                'en': 'The hub token',
+            },
+            'info': {
+                'zh': '该token可以在www.modelscope.cn找到',
+                'en': 'Find the token in www.modelscope.cn',
+            }
+        }
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='save_param', open=False):
+            with gr.Blocks():
+                with gr.Row():
+                    gr.Checkbox(elem_id='push_to_hub', scale=20)
+                    gr.Textbox(elem_id='hub_model_id', lines=1, scale=20)
+                    gr.Checkbox(elem_id='hub_private_repo', scale=20)
+                    gr.Dropdown(
+                        elem_id='hub_strategy',
+                        scale=20,
+                        choices=['end', 'every_save', 'checkpoint', 'all_checkpoints'])
+                    gr.Textbox(elem_id='hub_token', lines=1, scale=20)

ms-swift/swift/ui/llm_train/self_cog.py

@@ -0,0 +1,57 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import Type
+
+import gradio as gr
+
+from swift.ui.base import BaseUI
+
+
+class SelfCog(BaseUI):
+
+    group = 'llm_train'
+
+    locale_dict = {
+        'self_cognition': {
+            'label': {
+                'zh': '自我认知任务参数设置',
+                'en': 'Self cognition settings'
+            },
+        },
+        'self_cognition_sample': {
+            'label': {
+                'zh': '数据及采样条数',
+                'en': 'Dataset sample size'
+            },
+            'info': {
+                'zh': '设置数据集采样的条数',
+                'en': 'Set the dataset sample size'
+            }
+        },
+        'model_name': {
+            'label': {
+                'zh': '模型认知名称',
+                'en': 'Model name'
+            },
+            'info': {
+                'zh': '设置模型应当认知自己的名字, 格式为:中文名字 英文名字,中间以空格分隔',
+                'en': 'Set the name of the model think itself of, the format is Chinesename Englishname, split by space'
+            }
+        },
+        'model_author': {
+            'label': {
+                'zh': '模型作者',
+                'en': 'Model author'
+            },
+            'info': {
+                'zh': '设置模型认知的自己的作者, 格式为:中文作者 英文作者,中间以空格分隔',
+                'en': 'Set the author of the model, the format is Chineseauthor Englishauthor, split by space'
+            }
+        },
+    }
+
+    @classmethod
+    def do_build_ui(cls, base_tab: Type['BaseUI']):
+        with gr.Accordion(elem_id='self_cognition', open=False):
+            with gr.Row():
+                gr.Textbox(elem_id='model_name', scale=20, is_list=True)
+                gr.Textbox(elem_id='model_author', scale=20, is_list=True)

ms-swift/swift/utils/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from .env import (get_dist_setting, get_node_setting, get_pai_tensorboard_dir, is_deepspeed_enabled, is_dist,
+                  is_dist_ta, is_local_master, is_master, is_mp, is_mp_ddp, is_pai_training_job, torchacc_trim_graph,
+                  use_hf_hub, use_torchacc)
+from .import_utils import (is_liger_available, is_lmdeploy_available, is_megatron_available, is_swanlab_available,
+                           is_unsloth_available, is_vllm_ascend_available, is_vllm_available, is_wandb_available,
+                           is_xtuner_available)
+from .io_utils import JsonlWriter, append_to_jsonl, download_ms_file, get_file_mm_type, read_from_jsonl, write_to_jsonl
+from .logger import get_logger
+from .np_utils import get_seed, stat_array, transform_jsonl_to_df
+from .tb_utils import TB_COLOR, TB_COLOR_SMOOTH, plot_images, read_tensorboard_file, tensorboard_smoothing
+from .torch_utils import (Serializer, activate_parameters, find_all_linears, find_embedding, find_layers, find_norm,
+                          freeze_parameters, gc_collect, get_current_device, get_device, get_device_count,
+                          get_model_parameter_info, get_n_params_grads, init_process_group, safe_ddp_context,
+                          set_default_ddp_config, set_device, show_layers, time_synchronize)
+from .utils import (add_version_to_work_dir, check_json_format, copy_files_by_pattern, deep_getattr, find_free_port,
+                    get_env_args, import_external_file, lower_bound, parse_args, patch_getattr, read_multi_line,
+                    seed_everything, split_list, subprocess_run, test_time, upper_bound)

ms-swift/swift/utils/constants.py

@@ -0,0 +1,27 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+BIN_EXTENSIONS = [
+    '.*.bin',
+    '.*.ts',
+    '.*.pt',
+    '.*.data-00000-of-00001',
+    '.*.onnx',
+    '.*.meta',
+    '.*.pb',
+    '.*.index',
+]
+
+PEFT_TYPE_KEY = 'peft_type'
+SWIFT_TYPE_KEY = 'swift_type'
+DEFAULT_ADAPTER = 'default'
+
+
+class Invoke(object):
+    KEY = 'invoked_by'
+    THIRD_PARTY = 'third_party'
+    PRETRAINED = 'from_pretrained'
+    PIPELINE = 'pipeline'
+    TRAINER = 'trainer'
+    LOCAL_TRAINER = 'local_trainer'
+    PREPROCESSOR = 'preprocessor'
+    SWIFT = 'swift'

ms-swift/swift/utils/logger.py

@@ -0,0 +1,138 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import importlib.util
+import logging
+import os
+from contextlib import contextmanager
+from types import MethodType
+from typing import Optional
+
+from modelscope.utils.logger import get_logger as get_ms_logger
+
+
+# Avoid circular reference
+def _is_local_master():
+    local_rank = int(os.getenv('LOCAL_RANK', -1))
+    return local_rank in {-1, 0}
+
+
+init_loggers = {}
+
+# old format
+# formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+logger_format = logging.Formatter('[%(levelname)s:%(name)s] %(message)s')
+
+info_set = set()
+warning_set = set()
+
+
+def info_once(self, msg, *args, **kwargs):
+    hash_id = kwargs.get('hash_id') or msg
+    if hash_id in info_set:
+        return
+    info_set.add(hash_id)
+    self.info(msg)
+
+
+def warning_once(self, msg, *args, **kwargs):
+    hash_id = kwargs.get('hash_id') or msg
+    if hash_id in warning_set:
+        return
+    warning_set.add(hash_id)
+    self.warning(msg)
+
+
+def get_logger(log_file: Optional[str] = None, log_level: Optional[int] = None, file_mode: str = 'w'):
+    """ Get logging logger
+
+    Args:
+        log_file: Log filename, if specified, file handler will be added to
+            logger
+        log_level: Logging level.
+        file_mode: Specifies the mode to open the file, if filename is
+            specified (if filemode is unspecified, it defaults to 'w').
+    """
+    if log_level is None:
+        log_level = os.getenv('LOG_LEVEL', 'INFO').upper()
+        log_level = getattr(logging, log_level, logging.INFO)
+    logger_name = __name__.split('.')[0]
+    logger = logging.getLogger(logger_name)
+    logger.propagate = False
+    if logger_name in init_loggers:
+        add_file_handler_if_needed(logger, log_file, file_mode, log_level)
+        return logger
+
+    # handle duplicate logs to the console
+    # Starting in 1.8.0, PyTorch DDP attaches a StreamHandler <stderr> (NOTSET)
+    # to the root logger. As logger.propagate is True by default, this root
+    # level handler causes logging messages from rank>0 processes to
+    # unexpectedly show up on the console, creating much unwanted clutter.
+    # To fix this issue, we set the root logger's StreamHandler, if any, to log
+    # at the ERROR level.
+    for handler in logger.root.handlers:
+        if type(handler) is logging.StreamHandler:
+            handler.setLevel(logging.ERROR)
+
+    stream_handler = logging.StreamHandler()
+    handlers = [stream_handler]
+
+    is_worker0 = _is_local_master()
+
+    if is_worker0 and log_file is not None:
+        file_handler = logging.FileHandler(log_file, file_mode)
+        handlers.append(file_handler)
+
+    for handler in handlers:
+        handler.setFormatter(logger_format)
+        handler.setLevel(log_level)
+        logger.addHandler(handler)
+
+    if is_worker0:
+        logger.setLevel(log_level)
+    else:
+        logger.setLevel(logging.ERROR)
+
+    init_loggers[logger_name] = True
+
+    logger.info_once = MethodType(info_once, logger)
+    logger.warning_once = MethodType(warning_once, logger)
+    return logger
+
+
+logger = get_logger()
+ms_logger = get_ms_logger()
+
+logger.handlers[0].setFormatter(logger_format)
+ms_logger.handlers[0].setFormatter(logger_format)
+log_level = os.getenv('LOG_LEVEL', 'INFO').upper()
+if _is_local_master():
+    ms_logger.setLevel(log_level)
+else:
+    ms_logger.setLevel(logging.ERROR)
+
+
+@contextmanager
+def ms_logger_ignore_error():
+    ms_logger = get_ms_logger()
+    origin_log_level = ms_logger.level
+    ms_logger.setLevel(logging.CRITICAL)
+    try:
+        yield
+    finally:
+        ms_logger.setLevel(origin_log_level)
+
+
+def add_file_handler_if_needed(logger, log_file, file_mode, log_level):
+    for handler in logger.handlers:
+        if isinstance(handler, logging.FileHandler):
+            return
+
+    if importlib.util.find_spec('torch') is not None:
+        is_worker0 = int(os.getenv('LOCAL_RANK', -1)) in {-1, 0}
+    else:
+        is_worker0 = True
+
+    if is_worker0 and log_file is not None:
+        file_handler = logging.FileHandler(log_file, file_mode)
+        file_handler.setFormatter(logger_format)
+        file_handler.setLevel(log_level)
+        logger.addHandler(file_handler)

ms-swift/swift/utils/tb_utils.py

@@ -0,0 +1,72 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+from typing import Dict, List, Optional, Tuple
+
+import matplotlib.pyplot as plt
+from tensorboard.backend.event_processing.event_accumulator import EventAccumulator
+
+Item = Dict[str, float]
+TB_COLOR, TB_COLOR_SMOOTH = '#FFE2D9', '#FF7043'
+
+
+def read_tensorboard_file(fpath: str) -> Dict[str, List[Item]]:
+    if not os.path.isfile(fpath):
+        raise FileNotFoundError(f'fpath: {fpath}')
+    ea = EventAccumulator(fpath)
+    ea.Reload()
+    res: Dict[str, List[Item]] = {}
+    tags = ea.Tags()['scalars']
+    for tag in tags:
+        values = ea.Scalars(tag)
+        r: List[Item] = []
+        for v in values:
+            r.append({'step': v.step, 'value': v.value})
+        res[tag] = r
+    return res
+
+
+def tensorboard_smoothing(values: List[float], smooth: float = 0.9) -> List[float]:
+    norm_factor = 0
+    x = 0
+    res: List[float] = []
+    for i in range(len(values)):
+        x = x * smooth + values[i]  # Exponential decay
+        norm_factor *= smooth
+        norm_factor += 1
+        res.append(x / norm_factor)
+    return res
+
+
+def plot_images(images_dir: str,
+                tb_dir: str,
+                smooth_key: Optional[List[str]] = None,
+                smooth_val: float = 0.9,
+                figsize: Tuple[int, int] = (8, 5),
+                dpi: int = 100) -> None:
+    """Using tensorboard's data content to plot images"""
+    smooth_key = smooth_key or []
+    os.makedirs(images_dir, exist_ok=True)
+    fname = [fname for fname in os.listdir(tb_dir) if os.path.isfile(os.path.join(tb_dir, fname))][0]
+    tb_path = os.path.join(tb_dir, fname)
+    data = read_tensorboard_file(tb_path)
+
+    for k in data.keys():
+        _data = data[k]
+        steps = [d['step'] for d in _data]
+        values = [d['value'] for d in _data]
+        if len(values) == 0:
+            continue
+        _, ax = plt.subplots(1, 1, squeeze=True, figsize=figsize, dpi=dpi)
+        ax.set_title(k)
+        if len(values) == 1:
+            ax.scatter(steps, values, color=TB_COLOR_SMOOTH)
+        elif k in smooth_key:
+            ax.plot(steps, values, color=TB_COLOR)
+            values_s = tensorboard_smoothing(values, smooth_val)
+            ax.plot(steps, values_s, color=TB_COLOR_SMOOTH)
+        else:
+            ax.plot(steps, values, color=TB_COLOR_SMOOTH)
+        fpath = os.path.join(images_dir, k.replace('/', '_').replace('.', '_'))
+        plt.savefig(fpath, dpi=dpi, bbox_inches='tight')
+        plt.close()

ms-swift/swift/utils/torch_utils.py

@@ -0,0 +1,391 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import gc
+import hashlib
+import os
+import pickle
+import re
+import time
+import uuid
+from bisect import bisect_right
+from contextlib import contextmanager, nullcontext
+from typing import Callable, Dict, List, Optional, Tuple, Union
+
+import numpy as np
+import torch
+import torch.distributed as dist
+import torch.nn as nn
+from datasets.utils.filelock import FileLock
+from modelscope.hub.utils.utils import get_cache_dir
+from transformers.integrations import is_deepspeed_zero3_enabled
+from transformers.utils import is_torch_cuda_available, is_torch_mps_available, is_torch_npu_available
+
+from .env import get_dist_setting, is_dist, is_dist_ta, is_local_master, is_master
+from .logger import get_logger
+from .utils import deep_getattr
+
+logger = get_logger()
+
+
+def _find_local_mac() -> str:
+    mac = uuid.getnode()
+    mac_address = ':'.join(('%012x' % mac)[i:i + 2] for i in range(0, 12, 2))
+    return mac_address
+
+
+def get_n_params_grads(model) -> Tuple[List[int], List[int]]:
+    n_params, n_grads = [], []
+    for p in model.parameters():
+        if is_deepspeed_zero3_enabled():
+            import deepspeed
+            context = deepspeed.zero.GatheredParameters(p)
+        else:
+            context = nullcontext()
+        with context:
+            n_params.append(p.numel())
+            n_grads.append(p.numel() if p.requires_grad else 0)
+    return n_params, n_grads
+
+
+def get_model_parameter_info(model: nn.Module, name: Optional[str] = None) -> str:
+    n_params, n_grads = get_n_params_grads(model)
+    n_params = sum(n_params)
+    n_grads = sum(n_grads)
+    n_buffers = sum(p.numel() for p in model.buffers())
+
+    if name is None:
+        name = model.__class__.__name__
+
+    n_params /= 1e6
+    n_grads /= 1e6
+    n_buffers /= 1e6
+    s = (f'{name}: '
+         f'{n_params:.4f}M Params ({n_grads:.4f}M Trainable '
+         f'[{100 * n_grads / n_params:.4f}%]), '
+         f'{n_buffers:.4f}M Buffers.')
+    return s
+
+
+def find_sub_module(module: torch.nn.Module, module_name: str) -> List[torch.nn.Module]:
+    _modules = list()
+    for name, sub_module in module.named_modules():
+        if not name:
+            continue
+        if name.endswith(module_name):
+            _modules.append(sub_module)
+    return _modules
+
+
+def show_layers(model: nn.Module, max_lines: Optional[int] = 20) -> None:
+    named_p = list(model.named_parameters())
+    for i, (n, p) in enumerate(named_p):
+        if max_lines is not None and i >= max_lines:
+            logger.info('...')
+            break
+        logger.info(f'[{n}]: requires_grad={p.requires_grad}, dtype={p.dtype}, device={p.device}')
+
+
+def freeze_parameters(model: nn.Module,
+                      freeze_parameters_ratio: float,
+                      freeze_parameters: List[str],
+                      freeze_parameters_regex: Optional[str] = None) -> None:
+    if freeze_parameters_ratio > 0:
+        n_parameters = get_n_params_grads(model)[0]
+        n_parameters = np.array(n_parameters, dtype=np.int64)
+        n_freeze_parameters = int(np.sum(n_parameters) * freeze_parameters_ratio)
+        n_parameters_cs = np.cumsum(n_parameters)
+        idx = bisect_right(n_parameters_cs, n_freeze_parameters)
+        for _, p in zip(range(idx), model.parameters()):
+            p.requires_grad = False
+
+    if len(freeze_parameters) > 0:
+        for n, p in model.named_parameters():
+            for freeze_p in freeze_parameters:
+                if n.startswith(freeze_p):
+                    p.requires_grad = False
+
+    if freeze_parameters_regex is not None:
+        try:
+            pattern = re.compile(freeze_parameters_regex)
+        except re.error as e:
+            logger.warning(f"Invalid freeze_parameters_regex '{freeze_parameters_regex}': {e}")
+            return
+
+        for n, p in model.named_parameters():
+            if pattern.search(n):
+                p.requires_grad = False
+
+
+def activate_parameters(model: nn.Module,
+                        additional_trainable_parameters: List[str],
+                        trainable_parameters_regex: Optional[str] = None) -> None:
+    has_activate = False
+    if len(additional_trainable_parameters) > 0:
+        for n, p in model.named_parameters():
+            for additional_tp in additional_trainable_parameters:
+                if n.startswith(additional_tp):
+                    p.requires_grad = True
+                    has_activate = True
+        if not has_activate:
+            logger.warning('len(additional_trainable_parameters) > 0 but no parameters are activated. '
+                           f'additional_trainable_parameters: {additional_trainable_parameters}')
+
+    has_activate = False
+    if trainable_parameters_regex is not None:
+        try:
+            pattern = re.compile(trainable_parameters_regex)
+        except re.error as e:
+            logger.warning(f"Invalid trainable_parameters_regex '{trainable_parameters_regex}': {e}")
+            return
+
+        for n, p in model.named_parameters():
+            if pattern.search(n):
+                p.requires_grad = True
+                has_activate = True
+
+        if not has_activate:
+            logger.warning('trainable_parameters_regex is provided but no parameters are activated. '
+                           f'trainable_parameters_regex: {trainable_parameters_regex}')
+
+
+def time_synchronize() -> float:
+    torch.cuda.synchronize()
+    return time.perf_counter()  # second
+
+
+def _get_max_memory(device_ids: List[int]) -> Dict[Union[int, str], int]:
+    """add feat in accelerate to support MP + DDP"""
+    import psutil
+    # Make sure CUDA is initialized on each GPU to have the right memory info.
+    for i in device_ids:
+        _ = torch.tensor([0], device=i)
+
+    device_ids_set = set(device_ids)
+    max_memory = {}
+    for i in range(get_device_count()):
+        max_memory[i] = 0
+        if i in device_ids_set:
+            max_memory[i] = torch.cuda.mem_get_info(i)[0]
+    max_memory['cpu'] = psutil.virtual_memory().available
+    return max_memory
+
+
+def _sync_max_memory(max_memory: Dict[Union[int, str], int]) -> Dict[Union[int, str], int]:
+    """Make sure that the model structure of MP(device_map) is the same, when using DDP."""
+    max_memory_list = [v for k, v in max_memory.items() if (v > 0 and k != 'cpu')]
+    _, local_rank, world_size, _ = get_dist_setting()
+    src_tensor = torch.tensor(max_memory_list).to(local_rank)
+    tgt_tensor_list = [torch.zeros_like(src_tensor) for _ in range(world_size)]
+    dist.all_gather(tgt_tensor_list, src_tensor)
+    tgt_tensor = torch.stack(tgt_tensor_list, dim=0)
+    new_max_memory_iter = iter(tgt_tensor.min(dim=0)[0].tolist())
+    new_max_memory = {}
+    for k, v in max_memory.items():
+        new_max_memory[k] = v
+        if v > 0 and k != 'cpu':
+            new_max_memory[k] = next(new_max_memory_iter)
+    return new_max_memory
+
+
+def find_layers(
+    model: nn.Module,
+    cond: Callable[[str, nn.Module], bool],
+    sub_module: Optional[str] = None,
+    min_name_len: Optional[int] = None,
+) -> List[str]:
+    # The content of target_module_names cannot exist in inner_nodes.
+    sub_module_str = sub_module
+    if sub_module is None:
+        sub_module = model
+    else:
+        sub_module = deep_getattr(model, sub_module)
+    inner_nodes = set()
+    for name, module in model.named_modules():
+        name = re.sub(r'\d+\.', '{}.', name)
+        if not cond(name, module):
+            inner_nodes.add(name)
+    target_module_names = set()
+    for name, module in sub_module.named_modules():
+        if sub_module_str:
+            name = f'{sub_module_str}.{name}' if name else sub_module_str
+        if cond(name, module):
+            module_name_list = name.split('.')
+            module_name = module_name_list.pop()
+            i = 1
+            for inner_node in inner_nodes:
+                while module_name_list and inner_node.endswith(re.sub(
+                        r'\d+\.', '{}.', module_name)) or min_name_len and i < min_name_len:
+                    module_name = f'{module_name_list.pop()}.{module_name}'
+                    i += 1
+            target_module_names.add(module_name)
+    return list(target_module_names)
+
+
+def find_norm(model: nn.Module) -> List[str]:
+    # find_layer_norm
+    return find_layers(
+        model,
+        lambda name, module: isinstance(module, torch.nn.LayerNorm) or 'rmsnorm' in module.__class__.__name__.lower())
+
+
+def find_embedding(model: nn.Module) -> List[str]:
+    return find_layers(model, lambda name, module: isinstance(module, torch.nn.Embedding))
+
+
+def find_all_linears(model, model_arch=None, extra_layers=None, sub_module=None):
+    if model_arch is None:
+        from swift.llm import get_model_arch
+        model_arch = get_model_arch(model.model_meta.model_arch)
+    # lm_head
+    if model_arch and model_arch.lm_head:
+        output = model_arch.lm_head
+        idx = output.rfind('.')
+        lm_head_name = output[idx + 1:]
+    else:
+        lm_head_name = 'lm_head'
+    # 'score', 'classifier': classification model
+    # 'v_head': reward model
+    ignore_layers = [lm_head_name, 'score', 'v_head', 'classifier'] + ['lora_A', 'lora_B', 'base_layer']
+    ignore_linear_cls = [
+        'glulinear'  # phi4-mm
+    ]
+
+    def _cond(name, module):
+        module_name = module.__class__.__name__.lower()
+        if (extra_layers and isinstance(module, tuple(extra_layers)) or
+            ('linear' in module_name and all(linear_cls not in module_name
+                                             for linear_cls in ignore_linear_cls))) and all(layer not in name
+                                                                                            for layer in ignore_layers):
+            return True
+        return False
+
+    return find_layers(model, _cond, sub_module=sub_module)
+
+
+@contextmanager
+def safe_ddp_context(hash_id: Optional[str], use_barrier: bool = False):
+    if use_barrier and dist.is_initialized():
+        if is_dist() or is_dist_ta():
+            if not is_master():
+                dist.barrier()
+            if not is_local_master():
+                # Compatible with multi-machine scenarios,
+                # where each machine uses different storage hardware.
+                dist.barrier()
+        yield
+        if is_dist() or is_dist_ta():
+            if is_master():
+                dist.barrier()
+            if is_local_master():
+                dist.barrier()
+    elif hash_id is not None:
+        lock_dir = os.path.join(get_cache_dir(), 'lockers')
+        os.makedirs(lock_dir, exist_ok=True)
+        file_path = hashlib.sha256(hash_id.encode('utf-8')).hexdigest() + '.lock'
+        file_path = os.path.join(lock_dir, file_path)
+        with FileLock(file_path):
+            yield
+    else:
+        yield
+
+
+def get_device(local_rank: Optional[Union[str, int]] = None) -> str:
+    if local_rank is None:
+        local_rank = max(0, get_dist_setting()[1])
+    local_rank = str(local_rank)
+    if is_torch_npu_available():
+        device = 'npu:{}'.format(local_rank)
+    elif is_torch_mps_available():
+        device = 'mps:{}'.format(local_rank)
+    elif is_torch_cuda_available():
+        device = 'cuda:{}'.format(local_rank)
+    else:
+        device = 'cpu'
+
+    return device
+
+
+def get_current_device():
+    if is_torch_npu_available():
+        current_device = torch.npu.current_device()
+    elif is_torch_cuda_available():
+        current_device = torch.cuda.current_device()
+    elif is_torch_mps_available():
+        current_device = 'mps'
+    else:
+        current_device = 'cpu'
+    return current_device
+
+
+def set_device(local_rank: Optional[Union[str, int]] = None):
+    if local_rank is None:
+        local_rank = max(0, get_dist_setting()[1])
+    if is_torch_npu_available():
+        torch.npu.set_device(local_rank)
+    elif is_torch_cuda_available():
+        torch.cuda.set_device(local_rank)
+
+
+def get_device_count() -> int:
+    if is_torch_npu_available():
+        return torch.npu.device_count()
+    elif is_torch_cuda_available():
+        return torch.cuda.device_count()
+    else:
+        return 0
+
+
+def gc_collect() -> None:
+    gc.collect()
+    if is_torch_npu_available():
+        torch.npu.empty_cache()
+    elif is_torch_mps_available():
+        torch.mps.empty_cache()
+    elif is_torch_cuda_available():
+        torch.cuda.empty_cache()
+
+
+class Serializer:
+
+    @staticmethod
+    def to_tensor(obj):
+        res = pickle.dumps(obj)
+        res = np.array([len(res)], dtype=np.int64).tobytes() + res
+        res = np.frombuffer(res, dtype=np.uint8).copy()
+        res = torch.from_numpy(res)
+        return res
+
+    @staticmethod
+    def from_tensor(obj):
+        if isinstance(obj, torch.Tensor):
+            obj = obj.cpu().numpy()
+        res = obj.tobytes()
+        buffer_size = np.frombuffer(res[:8], dtype=np.int64)[0]
+        res = res[8:]
+        return pickle.loads(res[:buffer_size])
+
+
+def set_default_ddp_config():
+    # It runs normally with Python as well.
+    rank = int(os.getenv('RANK', -1))
+    if rank == -1:
+        os.environ['NPROC_PER_NODE'] = '1'
+        os.environ['RANK'] = '0'
+        os.environ['LOCAL_RANK'] = '0'
+        os.environ['WORLD_SIZE'] = '1'
+        os.environ['LOCAL_WORLD_SIZE'] = '1'
+        os.environ['MASTER_ADDR'] = '127.0.0.1'
+        os.environ['MASTER_PORT'] = os.environ.get('MASTER_PORT', '29500')
+
+
+def init_process_group(ddp_backend: Optional[str] = None):
+    if dist.is_initialized():
+        return
+    set_device()
+    if ddp_backend is None:
+        if is_torch_npu_available():
+            ddp_backend = 'hccl'
+        elif torch.cuda.is_available():
+            ddp_backend = 'nccl'
+        else:
+            ddp_backend = 'gloo'
+    dist.init_process_group(backend=ddp_backend)

ms-swift/tests/deploy/test_dataset.py

@@ -0,0 +1,61 @@
+def _test_client(port=8000):
+    import time
+    import aiohttp
+    from swift.llm import InferClient, InferRequest, RequestConfig, load_dataset, run_deploy
+    dataset = load_dataset(['AI-ModelScope/alpaca-gpt4-data-zh#1000'], num_proc=4)
+    infer_client = InferClient(port=port)
+    while True:
+        try:
+            infer_client.models
+            break
+        except Exception:
+            time.sleep(1)
+            pass
+    infer_requests = []
+    for data in dataset[0]:
+        infer_requests.append(InferRequest(**data))
+    request_config = RequestConfig(seed=42, max_tokens=256, temperature=0.8)
+
+    resp = infer_client.infer(infer_requests, request_config=request_config, use_tqdm=False)
+    print(len(resp))
+
+
+def _test(infer_backend):
+    import os
+    os.environ['CUDA_VISIBLE_DEVICES'] = '0'
+
+    from swift.llm import DeployArguments
+    from swift.llm import run_deploy
+    args = DeployArguments(model='Qwen/Qwen2-7B-Instruct', infer_backend=infer_backend, verbose=False)
+    with run_deploy(args) as port:
+        _test_client(port)
+
+
+def test_vllm():
+    _test('vllm')
+
+
+def test_lmdeploy():
+    _test('lmdeploy')
+
+
+def test_pt():
+    _test('pt')
+
+
+def test_vllm_origin():
+    import subprocess
+    import sys
+    from modelscope import snapshot_download
+    model_dir = snapshot_download('Qwen/Qwen2-7B-Instruct')
+    args = [sys.executable, '-m', 'vllm.entrypoints.openai.api_server', '--model', model_dir]
+    process = subprocess.Popen(args)
+    _test_client()
+    process.terminate()
+
+
+if __name__ == '__main__':
+    # test_vllm_origin()
+    # test_vllm()
+    test_lmdeploy()
+    # test_pt()