Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/benchmark-appendix.jpg filter=lfs diff=lfs merge=lfs -text
+assets/benchmark.jpg filter=lfs diff=lfs merge=lfs -text

LICENSE

@@ -0,0 +1,210 @@
+Tencent is pleased to support the open source community by making Tencent Hy4 preview available. 
+
+Copyright (C) 2026 Tencent.  All rights reserved. 
+
+Tencent Hy4 preview is licensed under the Apache-2.0.
+
+
+Terms of the Apache-2.0:
+--------------------------------------------------------------------
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

finetune/README.md

@@ -0,0 +1,385 @@
+<p align="left">
+    <a href="README_CN.md">中文</a> ｜ English
+</p>
+
+# Model Fine-tuning
+
+Hy4 preview provides processes related to model fine-tuning. This section details how to process training data for model fine-tuning purposes.
+
+## Training Data Format and Processing
+
+**Hy4 preview supports both "slow thinking" and "fast thinking" modes. You can control the mode via the `reasoning_effort` parameter (options: `high`, `no_think`).**
+
+The training data should be formatted as a list of messages. By default, the system prompt for both training and inference is empty, but you may customize it as needed.
+
+```python
+# Fast thinking pattern (no_think)
+{"reasoning_effort": "no_think", "messages": [{"content": "You are a helpful assistant.\nThe current time is 2026-01-01 13:26:12 Thursday", "role": "system"}, {"content": "1+1=?", "role": "user"}, {"role": "assistant", "content": "1+1=2"}]}
+
+# Slow thinking pattern (high)
+{"reasoning_effort": "high", "messages": [{"content": "You are a helpful assistant.\nThe current time is 2026-01-01 13:26:12 Thursday", "role": "system"}, {"content": "1+1=?", "role": "user"}, {"role": "assistant", "content": "1+1=2", "reasoning_content": "The user is asking for the result of 1 + 1. In basic decimal arithmetic, 1 + 1 equals 2."}]}
+```
+
+Example of using `apply_chat_template` to tokenize:
+
+```python
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("./models", use_fast=False, trust_remote_code=True)
+
+messages = [
+    {"content": "You are a helpful assistant.", "role": "system"},
+    {"content": "1+1=?", "role": "user"},
+    {"role": "assistant", "content": "1+1=2"}
+]
+ids = tokenizer.apply_chat_template(messages, tokenize=True, reasoning_effort="no_think")
+```
+
+## Fine-tuning Process
+
+### Hardware Requirements
+
+Based on testing, the minimum resource configuration is as follows:
+
+- **LoRA Fine-tuning**: At least 8 machines with 64 GPUs (at least 96GB GPU memory per GPU, at least 2TB CPU memory per machine).
+- **Full Fine-tuning**: At least 16 machines with 128 GPUs (at least 96GB GPU memory per GPU, at least 2TB CPU memory per machine).
+
+> Note: The above are minimum resource configurations; actual requirements increase with `max_seq_length`, batch size, etc.
+
+### Configure Passwordless SSH Login Between Machines (Multi-Machine Training)
+
+> If you only use single-machine training, you can skip this section.
+
+The following instructions use two machines as an example, with their IPs denoted as `${ip1}` and `${ip2}`. All steps should be performed inside the Docker container.
+
+First, configure passwordless SSH for each container on every machine:
+
+```sh
+ssh-keygen			# Generate id_rsa and id_rsa.pub for passwordless login
+ssh-keygen -t rsa -A    # Generate /etc/ssh/ssh_host_rsa_key and ssh_host_ecdsa_key for SSH listening
+/usr/sbin/sshd -p 36005 -o ListenAddress=0.0.0.0        # Start SSH listening
+echo "Port 36005" > ~/.ssh/config   # Set SSH connection port to 36005
+passwd root    # Set the root password to avoid monitoring platform alerts
+```
+
+Note: `36005` is an example port. You may use any available port, but ensure it is **open** and **not occupied by other processes**.
+
+Next, in each machine's container, execute:
+
+```sh
+cat ~/.ssh/id_rsa.pub
+```
+
+**Copy the output SSH public key and paste it into the `~/.ssh/authorized_keys` file, one key per line. This must be done on every machine.** In the end, the `~/.ssh/authorized_keys` file on each machine should be identical and contain the public keys of all machines.
+
+Please note that for multi-node training, the code executed on each node must be identical. It is recommended to mount a shared network drive. If this is not possible, you must manually copy the dataset, scripts, and code to the same directory on each machine.
+
+### Launch Methods
+
+This project provides three fine-tuning methods. You can choose based on your needs:
+
+- **DeepSpeed Native Fine-tuning** (based on HuggingFace Transformers Trainer): Located in the `deepspeed_support` directory
+- **LLaMA-Factory Fine-tuning**: Located in the `llama_factory_support` directory
+- **ms-swift Fine-tuning**: Located in the `ms_swift_support` directory
+
+#### DeepSpeed Native Fine-tuning
+
+Reference: [HuggingFace Transformers Trainer](https://huggingface.co/docs/transformers/main/en/main_classes/trainer)
+
+##### Single-Machine Fine-tuning
+
+In the `deepspeed_support` directory, execute:
+
+```sh
+pip install -r requirements.txt
+bash train.sh
+```
+
+##### Multi-Machine Fine-tuning
+
+To launch fine-tuning across multiple machines, please first complete the configuration in [Configure Passwordless SSH Login Between Machines](#configure-passwordless-ssh-login-between-machines-multi-machine-training), and ensure all machines are within the same cluster.
+
+Confirm that dependencies are installed (if not, run `pip install -r requirements.txt`), then add the following configuration at the beginning of `train.sh`:
+
+```shell
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
+IP_LIST=${IP_LIST:-"127.0.0.1"}
+```
+
+Note: If the `IP_LIST` environment variable is not set, replace `IP_LIST` with the IP list! The format is:
+```
+For a single IP:
+IP_LIST=${ip_1}
+
+For multiple IPs:
+IP_LIST=${ip_1},${ip_2}
+
+```
+
+Replace `${ip_1}` and `${ip_2}` with the actual IP addresses.
+
+Then, on the machine with `${ip1}`, execute `bash train.sh` in the `deepspeed_support/` directory. On first launch, you may see the following output:
+
+```ssh
+The authenticity of host '[ip]:36005 ([ip]:36005)' can't be established.
+ECDSA key fingerprint is xxxxxx.
+ECDSA key fingerprint is MD5:xxxxxx.
+Are you sure you want to continue connecting (yes/no)?
+```
+
+Type `yes` to continue.
+
+##### Key Parameters
+
+The key parameters in the script are as follows:
+
+- `--deepspeed`: Path to the DeepSpeed configuration file. Four default DeepSpeed configuration files are provided in the `deepspeed_support` folder: `ds_zero2_no_offload.json`, `ds_zero2_offload.json`, `ds_zero3_no_offload.json`, and `ds_zero3_offload.json`, with different ZeRO stages (ZeRO-2 / ZeRO-3) and offload strategies selectable based on available GPU memory and communication constraints.
+- `--model_name_or_path`: Path to the Hy4 preview HF pre-trained model weights to load, otherwise loading will fail.
+- `--tokenizer_name_or_path`: Path to the tokenizer folder, otherwise loading will fail.
+- `--train_data_file`: Path to the training file, which should be a jsonl file.
+- `--output_dir`: Output directory where logs, tensorboard files, and model weights will be stored.
+- `--per_device_train_batch_size`: Batch size per GPU.
+- `--gradient_accumulation_steps`: Number of gradient accumulation steps. The global batch size is `per_device_train_batch_size * gradient_accumulation_steps * dp_size`.
+- `--max_steps`: Total number of training steps.
+- `--save_steps`: Number of steps between saving checkpoints.
+- `--use_lora`: Whether to use LoRA training. Also accepts `--lora_rank`, `--lora_alpha`, and `--lora_dropout` parameters. By default, LoRA is applied to "q_proj", "k_proj", "v_proj", and "o_proj". To change this, modify the code. Note: **When using LoRA training, only the LoRA weights are saved, not the base model weights.**
+- `--make_moe_param_leaf_module`: When using ZeRO-3 with MoE training, treat the MoE module as a leaf module, i.e., its parameters are not partitioned by ZeRO-3. This option is expected to significantly increase memory usage.
+- `--gradient_checkpointing`: Enable gradient checkpointing.
+- `--learning_rate`: Maximum learning rate during training.
+- `--min_lr`: Minimum learning rate during training.
+- `--use_flash_attn`: Enable flash-attention for accelerated training.
+
+**Notes:**
+
+- To resume training from a previously saved checkpoint rather than loading pre-trained weights, specify `--resume_from_checkpoint` with the path to the checkpoint. Do not specify `--model_name_or_path`; this will load only the weights without the training state.
+- When resuming from a checkpoint, there may be minor differences in loss due to the randomness of some non-deterministic algorithms. This is normal. See: [HuggingFace Transformers Trainer Randomness](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#randomness)
+- When `--model_name_or_path` is specified, all model-related parameters will be ignored.
+- Samples within a batch are padded to the length of the longest sample in the batch, but the maximum length of each sample is `max_seq_length`. Any excess will be truncated.
+- If you see a warning about **linear layer** bias weights not being loaded, you can ignore it; Hy4 preview's linear layers (q_proj / k_proj / v_proj / o_proj, etc.) do not use bias. Note: the MoE router's `e_score_correction_bias` is a buffer and is auto-loaded by the training script, so please do not ignore its loading failure.
+
+##### What if GPU Memory is Insufficient?
+
+Reference: [DeepSpeed Configuration](https://www.deepspeed.ai/docs/config-json/)
+
+You can try modifying the DeepSpeed configuration by removing the `auto` attribute from the following parameters and reducing their values:
+
+- `stage3_param_persistence_threshold`
+- `stage3_prefetch_bucket_size`
+- `stage3_max_reuse_distance`
+
+#### LLaMA-Factory Fine-tuning
+
+If you are familiar with LLaMA-Factory, you may use it for fine-tuning. All scripts, code, and configuration files are archived in the `llama_factory_support` directory. Unless otherwise specified, all files mentioned below are located in this directory.
+
+##### Installation
+
+You can install LLaMA-Factory by downloading the source code from https://github.com/hiyouga/LLaMA-Factory/tree/main and following the instructions on the website.
+
+##### Configuration Files
+
+We provide sample LLaMA-Factory fine-tuning configuration files: `hy_v4_lora_sft.yaml` and `hy_v4_full_sft.yaml`, corresponding to LoRA fine-tuning and full fine-tuning respectively.
+
+Key parameters in the configuration files are as follows:
+
+**Model:**
+
+- `model_name_or_path`: Path to the Hy4 preview HF format pre-trained model weights
+- `trust_remote_code`: Whether to trust remote code; Hy4 preview requires this to be set to `true`
+
+**Training Method:**
+
+- `stage`: Training stage, currently `sft` (supervised fine-tuning)
+- `finetuning_type`: Fine-tuning type, either `full` (full fine-tuning) or `lora` (LoRA fine-tuning)
+- `deepspeed`: DeepSpeed configuration file path; `../deepspeed_support/ds_zero3_offload.json` is recommended for full fine-tuning
+- `fsdp` + `fsdp_config`: FSDP distributed strategy; recommended for LoRA fine-tuning (configuration is built into `hy_v4_lora_sft.yaml`); mutually exclusive with DeepSpeed
+
+> **Distributed Strategy Recommendations:**
+> - **FSDP**: Recommended for LoRA fine-tuning, good compatibility and simple configuration
+> - **DeepSpeed ZeRO-3 + Offload**: Recommended for full fine-tuning or memory-constrained scenarios
+
+**LoRA Parameters (only effective during LoRA fine-tuning):**
+
+- `lora_rank`: LoRA rank, default `64`
+- `lora_alpha`: LoRA alpha coefficient, default `128`
+- `lora_dropout`: LoRA dropout ratio, default `0.05`
+- `lora_target`: Target modules for LoRA, default `q_a_proj,q_b_proj,kv_a_proj_with_mqa,kv_b_proj,o_proj`
+
+**Dataset:**
+
+- `dataset_dir`: Dataset directory path
+- `dataset`: Dataset name, must be registered in `dataset_info.json` under `dataset_dir`
+- `template`: Chat template; Hy4 preview uses `hy_v4`
+- `cutoff_len`: Maximum sequence length; sequences exceeding this will be truncated. For LoRA fine-tuning, a smaller value is recommended to save memory
+- `max_samples`: Maximum number of samples per dataset
+- `overwrite_cache`: Whether to overwrite cached preprocessed datasets
+
+**Output:**
+
+- `output_dir`: Output directory where logs, TensorBoard files, and weights will be stored
+- `logging_steps`: Number of steps between logging
+- `save_steps`: Number of steps between saving checkpoints
+- `plot_loss`: Whether to plot the training loss curve
+- `overwrite_output_dir`: Whether to overwrite the existing output directory
+- `save_only_model`: Whether to save only model weights (excluding optimizer states, etc.)
+- `report_to`: Logging tool, options: `none`, `wandb`, `tensorboard`, `swanlab`, `mlflow`
+
+**Training Hyperparameters:**
+
+- `per_device_train_batch_size`: Batch size per GPU
+- `gradient_accumulation_steps`: Gradient accumulation steps; `per_device_train_batch_size * gradient_accumulation_steps * dp_size` equals the global batch size
+- `learning_rate`: Maximum learning rate; `1.0e-5` recommended for full fine-tuning, `2.0e-4` for LoRA fine-tuning
+- `num_train_epochs`: Number of training epochs
+- `lr_scheduler_type`: Learning rate scheduler type; `cosine_with_min_lr` is recommended
+- `lr_scheduler_kwargs.min_lr_rate`: Ratio of minimum to maximum learning rate; e.g., `0.1` means the minimum learning rate is 10% of the maximum
+- `warmup_steps`: Number of warmup steps
+- `bf16`: Whether to use BFloat16 mixed precision training
+- `gradient_checkpointing`: Whether to enable gradient checkpointing to save memory
+- `ddp_timeout`: Distributed training timeout (milliseconds)
+- `flash_attn`: Attention implementation; `auto` (automatic selection) or `sdpa` is recommended
+- `resume_from_checkpoint`: Resume training from a specified checkpoint path; set to `null` to start from scratch
+
+##### Launch Fine-tuning
+
+For multi-machine fine-tuning, please first complete the configuration in [Configure Passwordless SSH Login Between Machines](#configure-passwordless-ssh-login-between-machines-multi-machine-training) (single-machine fine-tuning can skip this step).
+
+Modify the following configuration at the beginning of `train_lf.sh`:
+
+```shell
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
+export IP_LIST=${IP_LIST:-"127.0.0.1"}
+```
+
+Note:
+1. If the `IP_LIST` environment variable is not set, replace `IP_LIST` with the IP list! The format is:
+```
+For a single IP:
+IP_LIST=${ip_1}
+
+For multiple IPs:
+IP_LIST=${ip_1},${ip_2}
+
+```
+Replace `${ip_1}` and `${ip_2}` with the actual IP addresses.
+
+2. To specify a fine-tuning configuration file, set the `YAML_FILE` environment variable. The default is `hy_v4_full_sft.yaml`. For example, to use the LoRA fine-tuning configuration:
+```shell
+export YAML_FILE=hy_v4_lora_sft.yaml
+```
+
+Then, on each machine, run the launch script in the `llama_factory_support/` directory:
+
+```shell
+bash train_lf.sh
+```
+
+#### ms-swift Fine-tuning
+
+If you are familiar with ms-swift, you may use it for fine-tuning. All scripts, code, and configuration files are archived in the `ms_swift_support` directory. Unless otherwise specified, all files mentioned below are located in this directory.
+
+##### Installation
+
+You can install ms-swift via pip:
+
+```sh
+pip install ms-swift
+```
+
+Or install from source: https://github.com/modelscope/ms-swift
+
+##### Fine-tuning Scripts and Configuration Files
+
+| Fine-tuning Method | Configuration File | Launch Script |
+|----------------|-------------------|---------------|
+| Full Fine-tuning | `hy_v4_full_sft.yaml` | `bash sft_train.sh` |
+| LoRA Fine-tuning | `hy_v4_lora_sft.yaml` | `bash sft_train_lora.sh` |
+
+##### About the eos_token_id Patch
+
+The `hy_v4_swift_patches.py` file in the directory fixes an issue with the eos token in ms-swift's default template. The default template uses the `<｜hy_eos｜>` string as `chat_sep` and `suffix`, which gets tokenized into multiple token IDs, causing `model.generate()` to fail to stop correctly during inference.
+
+The patch re-registers the template using the `[['eos_token_id']]` syntax, allowing ms-swift to dynamically resolve `tokenizer.eos_token_id` at runtime and generate the correct single token.
+
+The launch script automatically loads this patch via `--custom_register_path hy_v4_swift_patches.py`, requiring no additional action.
+
+##### Key Parameters
+
+Key parameters in the configuration files are as follows:
+
+**Model:**
+
+- `model`: Model path, can be a HuggingFace Hub ID or a local path
+- `model_type`: Model type, set to `hy_v4`
+- `template`: Chat template, set to `hy_v4`
+- `torch_dtype`: Data type, `bfloat16` is recommended
+- `attn_impl`: Attention implementation, `sdpa` is recommended
+
+**Training Method:**
+
+- `train_type`: Fine-tuning type; set to `full` for full fine-tuning, `lora` for LoRA fine-tuning
+- `lora_rank`: LoRA rank, default `64`
+- `lora_alpha`: LoRA alpha coefficient, default `128`
+- `lora_dropout`: LoRA dropout ratio, default `0.05`
+
+**Dataset:**
+
+- `dataset`: Dataset path, supports local jsonl files (sharegpt format)
+- `max_length`: Maximum sequence length; sequences exceeding this will be truncated
+- `lazy_tokenize`: Whether to use lazy tokenization, `true` is recommended
+
+**Output:**
+
+- `output_dir`: Output directory
+- `save_steps`: Number of steps between saving checkpoints
+- `save_total_limit`: Maximum number of checkpoints to keep
+- `logging_steps`: Number of steps between logging
+- `report_to`: Logging tool, options: `none`, `wandb`, `tensorboard`, `swanlab`, `mlflow`
+
+**Training Hyperparameters:**
+
+- `per_device_train_batch_size`: Batch size per GPU
+- `gradient_accumulation_steps`: Gradient accumulation steps
+- `learning_rate`: Maximum learning rate; `1.0e-5` recommended for full fine-tuning, `2.0e-4` for LoRA fine-tuning
+- `num_train_epochs`: Number of training epochs
+- `lr_scheduler_type`: Learning rate scheduler type, `cosine` is recommended
+- `warmup_steps`: Number of warmup steps
+- `bf16`: Whether to use BFloat16 mixed precision training
+
+**Distributed Strategy / Optimization:**
+
+- `deepspeed`: DeepSpeed strategy, options: `zero0`, `zero2`, `zero2_offload`, `zero3`, `zero3_offload`; `zero3_offload` recommended for full fine-tuning
+- `fsdp` + `fsdp_config`: FSDP distributed strategy; recommended for LoRA fine-tuning; mutually exclusive with DeepSpeed
+- `gradient_checkpointing`: Whether to enable gradient checkpointing
+- `max_grad_norm`: Gradient clipping threshold
+
+> **Distributed Strategy Recommendations:**
+> - **FSDP**: Recommended for LoRA fine-tuning, good compatibility and simple configuration
+> - **DeepSpeed ZeRO-3 + Offload**: Recommended for full fine-tuning or memory-constrained scenarios
+
+**Other:**
+
+- `ddp_timeout`: Distributed training timeout (milliseconds)
+- `seed`: Random seed
+- `resume_from_checkpoint`: Resume training from a specified checkpoint path
+
+##### Launch Fine-tuning
+
+For multi-machine fine-tuning, please first complete the configuration in [Configure Passwordless SSH Login Between Machines](#configure-passwordless-ssh-login-between-machines-multi-machine-training) (single-machine fine-tuning can skip this step).
+
+Modify the following configuration in the `sft_train.sh` script:
+
+```shell
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
+export IP_LIST=${IP_LIST:-"127.0.0.1"}
+```
+
+Then, on each machine, execute the launch script in the `ms_swift_support/` directory:
+
+```sh
+# Single-machine training
+bash sft_train.sh
+
+# Multi-machine training (execute on each machine)
+IP_LIST="10.0.0.1,10.0.0.2" bash sft_train.sh
+```

finetune/README_CN.md

@@ -0,0 +1,384 @@
+<p align="left">
+   <a href="README.md">English</a> ｜ 中文
+</p>
+
+# 模型微调
+
+Hy4 preview 提供了模型微调相关流程，您可以在此章节对训练数据格式进行处理以供模型微调使用。
+
+## 训练数据格式及处理
+
+**Hy4 preview 同时支持慢思考与快思考两种模式，可通过 `reasoning_effort` 参数控制（可选值：`high`、`no_think`）。**
+
+训练数据按照以下形式处理为 messages 格式，训练和推理的默认 system prompt 为空，可以根据自己的需求进行设定。
+
+```python
+# Fast thinking pattern (no_think)
+{"reasoning_effort": "no_think", "messages": [{"content": "你是一个有用的人工智能助手。\n现在的时间是2026-01-01 13:26:12 周四", "role": "system"}, {"content": "1+1=?", "role": "user"}, {"role": "assistant", "content": "1+1=2"}]}
+
+# Slow thinking pattern (high)
+{"reasoning_effort": "high", "messages": [{"content": "你是一个有用的人工智能助手。\n现在的时间是2026-01-01 13:26:12 周四", "role": "system"}, {"content": "1+1=?", "role": "user"}, {"role": "assistant", "content": "1+1=2", "reasoning_content": "用户问的是1+1等于多少。在基本的十进制算术中，1+1等于2。"}]}
+```
+
+使用 `apply_chat_template` 进行 tokenize 的示例：
+
+```python
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained("./models", use_fast=False, trust_remote_code=True)
+
+messages = [
+    {"content": "你是一个有用的人工智能助手。", "role": "system"},
+    {"content": "1+1=?", "role": "user"},
+    {"role": "assistant", "content": "1+1=2"}
+]
+ids = tokenizer.apply_chat_template(messages, tokenize=True, reasoning_effort="no_think")
+```
+
+## 微调流程
+
+### 硬件需求
+
+经过测试，最小的资源配置如下：
+
+- **LoRA 微调**：最少需要 8 机 64 卡，每卡显存至少 96GB，每机 CPU 内存至少 2TB。
+- **全量微调**：最少需要 16 机 128 卡，每卡显存至少 96GB，每机 CPU 内存至少 2TB。
+
+> 注：以上为最小资源配置，实际所需资源会随 max_seq_length、batch size 等因素相应增加。
+
+### 配置机器间免密 ssh 登录（多机训练）
+
+> 如果只使用单机训练，可跳过本节。
+
+以下操作以两个机器为例，两台机器的 ip 分别以`${ip1}`和`${ip2}`标识，以下操作均在 docker container 内执行。
+
+首先，配置多机container免密，在每台机器上执行。
+
+```sh
+ssh-keygen			# 生成id_rsa和id_rsa.pub，用于免密登录
+ssh-keygen -t rsa -A    # 生成/etc/ssh/ssh_host_rsa_key和ssh_host_ecdsa_key， 用于后面启动ssh listen
+/usr/sbin/sshd -p 36005 -o ListenAddress=0.0.0.0        # 启动 SSH 监听
+echo "Port 36005" > ~/.ssh/config   # ssh 连接端口修改为 36005
+passwd root    # 需要配置root密码，否则监测平台会报警
+```
+
+注意：这里的`36005`是一个示例端口，可以选用任意端口，但需要保证使用的端口**开放**且**不被其他的进程占用**。
+
+接下来，在每台机器的 container 内，执行：
+
+```sh
+cat ~/.ssh/id_rsa.pub
+```
+
+**将输出的 ssh 公钥复制并粘贴到`~/.ssh/authorized_keys`文件中，每行一个公钥，每台机器上都要做这个操作**。最终每台机器上的`~/.ssh/authorized_keys`文件内容应当是一致的，并且包含了所有机器的公钥。
+
+需要注意，多节点训练时，每个节点上执行的代码都得一致，建议挂载一个共享的网络盘，如果无法挂载共享网盘，则需要手动将数据集、脚本、代码复制在多台机器的相同目录下。
+
+### 启动方式
+
+本项目提供三种微调方式，您可以根据需求选择：
+
+- **DeepSpeed 原生微调**（基于 HuggingFace Transformers Trainer）：位于 `deepspeed_support` 目录下
+- **LLaMA-Factory 微调**：位于 `llama_factory_support` 目录下
+- **ms-swift 微调**：位于 `ms_swift_support` 目录下
+
+#### DeepSpeed 原生微调
+
+参考：[HuggingFace Transformers Trainer](https://huggingface.co/docs/transformers/main/en/main_classes/trainer)
+
+##### 单机启动微调
+
+在 `deepspeed_support` 目录下，执行：
+
+```sh
+pip install -r requirements.txt
+bash train.sh
+```
+
+##### 多机启动微调
+
+如果要用多台机器启动微调，请先完成 [配置机器间免密 ssh 登录](#配置机器间免密-ssh-登录多机训练) 中的配置，并保证多台机器在一个集群内。
+
+确认依赖已经安装完成（如未安装，请执行`pip install -r requirements.txt`安装），然后在`train.sh`中的开头增加以下配置：
+
+```shell
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
+IP_LIST=${IP_LIST:-"127.0.0.1"}
+```
+
+注意：如果`IP_LIST`环境变量未设置，则将`IP_LIST`替换为IP列表！格式为：
+```
+如果只有一个IP：
+IP_LIST=${ip_1}
+
+如果有多个IP：
+IP_LIST=${ip_1},${ip_2}
+
+```
+
+请将`${ip_1}`和`${ip_2}`替换为真实的IP地址。
+
+��后，在`${ip1}`的机器上，在`deepspeed_support/`目录下，执行`bash train.sh`即可，注意第一次启动时可能会看见以下的输出：
+
+```ssh
+The authenticity of host '[ip]:36005 ([ip]:36005)' can't be established.
+ECDSA key fingerprint is xxxxxx.
+ECDSA key fingerprint is MD5:xxxxxx.
+Are you sure you want to continue connecting (yes/no)?
+```
+
+此时输入`yes`即可继续。
+
+##### 关键参数
+
+脚本中的关键参数如下：
+
+- `--deepspeed`: 此参数应当指向一个 deepspeed 的配置文件，`deepspeed_support`文件夹下提供了四种 DeepSpeed 的默认配置文件：`ds_zero2_no_offload.json`, `ds_zero2_offload.json`, `ds_zero3_no_offload.json`, `ds_zero3_offload.json`，可根据显存与通信情况选择不同的 ZeRO 阶段（ZeRO-2 / ZeRO-3）与 offload 策略
+- `--model_name_or_path`: 要加载的 Hy4 preview 的 HF 预训练模型权重，否则无法加载
+- `--tokenizer_name_or_path`: tokenizer 文件夹路径, 否则无法加载
+- `--train_data_file`: 训练文件路径，应该为一个 jsonl 文件
+- `--output_dir`: 输出文件夹，log、tensorboard 和权重都会存储在这个路径下
+- `--per_device_train_batch_size`: 每张卡上的 batch size
+- `--gradient_accumulation_steps`: 梯度累计次数，`per_device_train_batch_size * gradient_accumulation_steps * dp_size`为 global_batch_size
+- `--max_steps`: 训练的总步数
+- `--save_steps`: 每多少个 step 存储一个 checkpoint
+- `--use_lora`: 是否用 lora 训练，同时接收`--lora_rank`，`--lora_alpha`和`--lora_dropout`参数。lora 默认应用于 MLA（Multi-head Latent Attention）投影层："q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"，如果需要改变的话在代码中修改即可。注意：**使用 lora 训练时，只会保存 lora 的权重，而不会保存 base 模型的权重**
+- `--make_moe_param_leaf_module`：当用 zero3 以及 MoE 训练时，将 MoE 模块视作一个 leaf module，即它的参数不进行 zero3 切分，这个选项预计会显著增加显存占用
+- `--gradient_checkpointing`：开启梯度检查点- `--learning_rate`: 训练时的最大学习率
+- `--min_lr`: 训练时的最小学习率
+- `--use_flash_attn`: 开启 flash-attention 进行训练加速
+
+**注意：**
+
+- 如果想从一个中途保存的 ckpt 继续训练，而不是加载一个预训练的权重，直接指定`--resume_from_checkpoint`为之前训练保存的 ckpt 路径，不要指定`--model_name_or_path`，这样只会加载权重，而不会加载训练状态
+- 从 ckpt 继续训练时，loss 可能会有微小的偏差，这是由一些非确定性算法带来的随机性，是正常现象。参考：[HuggingFace Transformers Trainer Randomness](https://huggingface.co/docs/transformers/main/en/main_classes/trainer#randomness)
+- 当 `--model_name_or_path` 有效时，所有模型相关的参数都会被忽略
+- 一个 batch 内的样本会通过 padding 对齐 batch 内最长的样本，而每条样本的长度最长为 max_seq_length，超出的部分会被裁剪
+- 如果报出**线性层** bias 权重没有 load 的 warning，忽略即可，Hy4 preview 的线性层（q_a_proj / q_b_proj / kv_a_proj_with_mqa / kv_b_proj / o_proj 等）不使用 bias。注意：MoE 路由的 `e_score_correction_bias` 属于 buffer，已由训练脚本自动加载，如果加载失败请不要忽略。
+
+##### 显存不足怎么办？
+
+参考：[DeepSpeed Configuration](https://www.deepspeed.ai/docs/config-json/)
+
+可以尝试修改 ds config，去掉这几个参数的 auto 属性，改小试试看：
+
+- `stage3_param_persistence_threshold`
+- `stage3_prefetch_bucket_size`
+- `stage3_max_reuse_distance`
+
+#### LLaMA-Factory 微调
+
+如果对 LLaMA-Factory 较为熟悉，可使用 LLaMA-Factory 进行微调。脚本、代码以及配置文件都归档在 `llama_factory_support` 目录下。如果没有特别说明，接下来我们提到的文件都是该目录下的文件。
+
+##### 安装
+
+可以通过下载源码 https://github.com/hiyouga/LLaMA-Factory/tree/main ，根据网站的指引进行安装。
+
+##### 配置文件
+
+我们提供了 llama-factory 的微调示例配置文件 `hy_v4_lora_sft.yaml`和`hy_v4_full_sft.yaml`文件，分别对应 LoRA 微调和全量微调。
+
+脚本中的关键参数如下：
+
+**模型相关：**
+
+- `model_name_or_path`: Hy4 preview HF 格式预训练模型权重路径
+- `trust_remote_code`: 是否信任远程代码, Hy4 preview 需要设置为 `true`
+
+**训练方法：**
+
+- `stage`: 训练阶段, 当前为 `sft`(监督微调)
+- `finetuning_type`: 微调类型, 可选 `full`(全量微调) 或 `lora`(LoRA 微调)
+- `deepspeed`: DeepSpeed 配置文件路径, 全量微调推荐 `../deepspeed_support/ds_zero3_offload.json`
+- `fsdp` + `fsdp_config`: FSDP 分布式策略, LoRA 微调推荐使用 FSDP（配置已内置在 `hy_v4_lora_sft.yaml` 中）; 与 DeepSpeed 二选一
+
+> **分布式策略选择建议：**
+> - **FSDP**：推荐用于 LoRA 微调，兼容性好，配置简单
+> - **DeepSpeed ZeRO-3 + Offload**：推荐用于全量微调或显存紧张的场景
+
+**LoRA 参数(仅 LoRA 微调时生效)：**
+
+- `lora_rank`: LoRA 秩, 默认 `64`
+- `lora_alpha`: LoRA alpha 系数, 默认 `128`
+- `lora_dropout`: LoRA dropout 比率, 默认 `0.05`
+- `lora_target`: LoRA 应用的目标模块, 默认为 `q_a_proj,q_b_proj,kv_a_proj_with_mqa,kv_b_proj,o_proj`
+
+**数据集：**
+
+- `dataset_dir`: 数据集目录路径
+- `dataset`: 数据集名称, 需要在 `dataset_dir` 下的 `dataset_info.json` 中注册
+- `template`: 对话模板, Hy4 preview 使用 `hy_v4`
+- `cutoff_len`: 最大序列长度, 超出部分会被截断; LoRA 微调建议适当减小以节省显存
+- `max_samples`: 每个数据集最多使用的样本数
+- `overwrite_cache`: 是否覆盖已缓存的预处理数据集
+
+**输出：**
+
+- `output_dir`: 输出目录, 日志、TensorBoard 和权重都会存储在此路径下
+- `logging_steps`: 每多少步记录一次日志
+- `save_steps`: 每多少步保存一次 checkpoint
+- `plot_loss`: 是否绘制训练 loss 曲线
+- `overwrite_output_dir`: 是否覆盖已有的输出目录
+- `save_only_model`: 是否只保存模型权重(不保存优化器状态等)
+- `report_to`: 日志上报工具, 可选 `none`, `wandb`, `tensorboard`, `swanlab`, `mlflow`
+
+**训练超参数：**
+
+- `per_device_train_batch_size`: 每张卡上的 batch size
+- `gradient_accumulation_steps`: 梯度累积步数, `per_device_train_batch_size * gradient_accumulation_steps * dp_size` 为 global batch size
+- `learning_rate`: 最大学习率, 全量微调推荐 `1.0e-5`, LoRA 微调推荐 `2.0e-4`
+- `num_train_epochs`: 训练轮数
+- `lr_scheduler_type`: 学习率调度器类型, 推荐使用 `cosine_with_min_lr`
+- `lr_scheduler_kwargs.min_lr_rate`: 最小学习率与最大学习率的比值, 例如 `0.1` 表示最小学习率为最大学习率的 10%
+- `warmup_steps`: 预热步数
+- `bf16`: 是否使用 BFloat16 混合精度训练
+- `gradient_checkpointing`: 是否开启梯度重计算以节省显存
+- `ddp_timeout`: 分布式训练超时时间(毫秒)
+- `flash_attn`: 注意力实现方式, 推荐 `auto`(自动选择) 或 `sdpa`
+- `resume_from_checkpoint`: 从指定 checkpoint 路径恢复训练, 设为 `null` 表示从头开始训练
+
+##### 启动微调
+
+如需多机训练，请先完成 [配置机器间免密 ssh 登录](#配置机器间免密-ssh-登录多机训练) 中的配置（单机训练可跳过此步骤）。
+
+修改`train_lf.sh`中开头的以下配置：
+
+```shell
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
+export IP_LIST=${IP_LIST:-"127.0.0.1"}
+```
+
+注意：
+1. 如果`IP_LIST`环境变量未设置，则将`IP_LIST`替换为IP列表！格式为：
+```
+如果只有一个IP：
+IP_LIST=${ip_1}
+
+如果有多个IP：
+IP_LIST=${ip_1},${ip_2}
+
+```
+请将`${ip_1}`和`${ip_2}`替换为真实的IP地址。
+
+2. 如需指定微调配置文件，可通过`YAML_FILE`环境变量设置，默认为`hy_v4_full_sft.yaml`。例如使用 LoRA 微调配置：
+```shell
+export YAML_FILE=hy_v4_lora_sft.yaml
+```
+
+然后，在每一台机器上，在`llama_factory_support/`目录下执行启动脚本：
+
+```shell
+bash train_lf.sh
+```
+
+#### ms-swift 微调
+
+如果对 ms-swift 较为熟悉，可使用 ms-swift 进行微调。脚本、代码以及配置文件都归档在 `ms_swift_support` 目录下。如果没有特别说明，接下来我们提到的文件都是该目录下的文件。
+
+##### 安装
+
+可以通过 pip 安装 ms-swift：
+
+```sh
+pip install ms-swift
+```
+
+或从源码安装：https://github.com/modelscope/ms-swift
+
+##### 微调脚本与配置文件
+
+| 微调方式 | 配置文件 | 启动脚本 |
+|---------|---------|---------|
+| 全量微调 | `hy_v4_full_sft.yaml` | `bash sft_train.sh` |
+| LoRA 微调 | `hy_v4_lora_sft.yaml` | `bash sft_train_lora.sh` |
+
+##### 关于 eos_token_id Patch
+
+目录下的 `hy_v4_swift_patches.py` 文件用于修复 ms-swift 默认模板中 eos token 的问题。默认模板将 `<｜hy_eos｜>` 字符串作为 `chat_sep` 和 `suffix`，该字符串会被 tokenize 为多个 token ID，导致推理时 `model.generate()` 无法正确停止。
+
+Patch 通过 `[['eos_token_id']]` 语法重新注册模板，使 ms-swift 在运行时动态解析 `tokenizer.eos_token_id`，生成正确的单个 token。
+
+启动脚本已通过 `--custom_register_path hy_v4_swift_patches.py` 自动加载此 patch，无需额外操作。
+
+##### 关键参数
+
+配置文件中的关键参数如下：
+
+**模型相关：**
+
+- `model`: 模型路径，可以是 HuggingFace Hub ID 或本地路径
+- `model_type`: 模型类型，设为 `hy_v4`
+- `template`: 对话模板，设为 `hy_v4`
+- `torch_dtype`: 数据类型，推荐 `bfloat16`
+- `attn_impl`: 注意力实现，推荐 `sdpa`
+
+**训练方法：**
+
+- `train_type`: 微调类型，全量微调设为 `full`，LoRA 微调设为 `lora`
+- `lora_rank`: LoRA 秩，默认 `64`
+- `lora_alpha`: LoRA alpha 系数，默认 `128`
+- `lora_dropout`: LoRA dropout 比率，默认 `0.05`
+
+**数据集：**
+
+- `dataset`: 数据集路径，支持本地 jsonl 文件（sharegpt 格式）
+- `max_length`: 最大序列长度，超出部分会被截断
+- `lazy_tokenize`: 是否延迟 tokenize，推荐 `true`
+
+**输出：**
+
+- `output_dir`: 输出目录
+- `save_steps`: 每多少步保存一次 checkpoint
+- `save_total_limit`: 最多保留的 checkpoint 数量
+- `logging_steps`: 每多少步记录一次日志
+- `report_to`: 日志上报工具，可选 `none`, `wandb`, `tensorboard`, `swanlab`, `mlflow`
+
+**训练超参数：**
+
+- `per_device_train_batch_size`: 每张卡上的 batch size
+- `gradient_accumulation_steps`: 梯度累积步数
+- `learning_rate`: 最大学习率，全量微调推荐 `1.0e-5`，LoRA 微调推荐 `2.0e-4`
+- `num_train_epochs`: 训练轮数
+- `lr_scheduler_type`: 学习率调度器类型，推荐 `cosine`
+- `warmup_steps`: 预热步数
+- `bf16`: 是否使用 BFloat16 混合精度训练
+
+**分布式策略 / 优化：**
+
+- `deepspeed`: DeepSpeed 策略，可选 `zero0`, `zero2`, `zero2_offload`, `zero3`, `zero3_offload`；全量微调推荐 `zero3_offload`
+- `fsdp` + `fsdp_config`: FSDP 分布式策略，LoRA 微调推荐使用 FSDP；与 DeepSpeed 二选一
+- `gradient_checkpointing`: 是否开启梯度重计算
+- `max_grad_norm`: 梯度裁剪阈值
+
+> **分布式策略选择建议：**
+> - **FSDP**：推荐用于 LoRA 微调，兼容性好，配置简单
+> - **DeepSpeed ZeRO-3 + Offload**：推荐用于全量微调或显存紧张的场景
+
+**其他：**
+
+- `ddp_timeout`: 分布式训练超时时间（毫秒）
+- `seed`: 随机种子
+- `resume_from_checkpoint`: 从指定 checkpoint 路径恢复训练
+
+##### 启动微调
+
+如需多机训练，请先完成 [配置机器间免密 ssh 登录](#配置机器间免密-ssh-登录多机训练) 中的配置（单机训练可跳过此步骤）。
+
+修改 `sft_train.sh` 脚本中的以下配置：
+
+```shell
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
+export IP_LIST=${IP_LIST:-"127.0.0.1"}
+```
+
+然后，在每一台机器上，在 `ms_swift_support/` 目录下执行启动脚本：
+
+```sh
+# 单机训练
+bash sft_train.sh
+
+# 多机训练（在每台机器上执行）
+IP_LIST="10.0.0.1,10.0.0.2" bash sft_train.sh
+```

finetune/data/example_data.jsonl

@@ -0,0 +1,8 @@
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "能否帮我撰写一个关于环保的议论文开头部分？\n主题：环保\n立场：重要性和我们的责任"}, {"role": "assistant", "content": "在我们的生活中，环保已经成为一个不可忽视的议题。随着科技的发展和人口的增长，我们的环境正在遭受前所未有的压力。森林被砍伐，河流被污染，动植物种类正在消失，全球气候正在发生变化。这些都是我们无法忽视的事实。然而，环保并不仅仅是一个环境问题，它更是一个关乎我们生存和发展的问题。环保的重要性已经超越了任何一个单一的领域，它关乎我们的健康、经济、社会和文化。因此，我们每一个人都有责任和义务去保护我们的环境，这不仅是为了我们自己，也是为了我们的后代和整个地球。"}]}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "请描述小红书APP的用户画像，并用表格的形式展示结果。"}, {"role": "assistant", "content": "| 用户特征   | 描述                           |\n| -------- | ------------------------------ |\n| 年龄段    | 18-35岁                        |\n| 性别      | 女性用户较多（约占70%）          |\n| 地域      | 一线城市和二线城市用户占比较高        |\n| 职业      | 白领、创业者、自由职业者等           |\n| 爱好      | 美容、时尚、旅行、美食、健康、生活方式等 |\n| 消费能力  | 中高消费能力用户                  |\n| 使用目的  | 阅读内容，分享购物心得，结识相似兴趣的朋友  |\n\n注：以上用户画像仅供参考，实际情况可能会有所不同。"}]}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "请根据以下两个句子的描述，判断它们是否对同一情况进行了描述。句子1：“外面下着大雨，人们纷纷打起了伞。” 句子2：“由于雨量较大，人们都撑起了雨伞。”"}, {"role": "assistant", "content": "是的，这两个句子描述的是同一情况。"}]}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "你认为游戏开发者应该如何处理游戏中的社交网络集成？"}, {"role": "assistant", "content": "游戏开发者在处理游戏中的社交网络集成时可以考虑以下几点：\n\n1. 以用户为中心：确保集成社交网络功能是为了提高用户体验，让玩家更容易地与朋友互动、分享成就和游戏进度。\n\n2. 保护隐私：在集成社交网络时，务必保护用户的隐私。在收集和使用用户数据时，需要遵循相关法规和用户协议，同时提供明确的隐私政策。\n\n3. 适度推送：在推送游戏动态、邀请和通知时要适度，避免过度打扰用户或侵犯用户隐私。可提供自定义设置，让玩家选择性地接收通知。\n\n4. 社交功能设计：在游戏中加入社交功能，如聊天、组队、比赛等，以增进玩家之间的互动。同时，确保这些功能易用且不影响游戏体验。\n\n5. 社区建设：鼓励玩家在社交平台上建立游戏相关的社区，分享游戏心得、攻略和经验。这将有助于增加游戏的知名度和吸引力。\n\n6. 奖励机制：设定奖励机制，鼓励玩家在社交网络上分享游戏内容、邀请好友等。但要确保奖励机制不会破坏游戏平衡。\n\n7. 持续更新和优化：根据玩家的反馈，不断优化社交网络集成功能，确保其与游戏的其他方面保持一致。\n\n总之，游戏开发者应该在保护用户隐私的前提下，提供便捷的社交网络集成功能，以提高游戏的互动性和吸引力。同时，要关注玩家的反馈，不断优化功能。"}]}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "将这段过去进行时的文字改为现在完成进行时：“她们在公园里跳舞。”"}, {"role": "assistant", "content": "她们已经在公园里跳舞了。"}]}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "在godaddy上买了一台服务器怎么更改root账户"}, {"role": "assistant", "content": "在GoDaddy上购买的服务器可能是Linux系统的VPS或者是专用服务器。要更改root账户，你可以通过SSH连接到服务器并修改密码。以下是具体步骤：\n\n1. 打开终端（Terminal）或使用SSH客户端（如PuTTY）。\n2. 使用SSH连接到服务器。在终端中输入以下命令：\n\n   ```\n   ssh root@your_server_ip_address\n   ```\n\n   将`your_server_ip_address`替换为你的服务器IP地址。如果需要，你可能还需要输入服务器的SSH端口号。\n   \n3. 当系统提示输入密码时，输入你的当前root密码。如果连接成功，你将看到服务器的命令提示符。\n\n4. 要更改root密码，请输入以下命令：\n\n   ```\n   passwd\n   ```\n\n5. 系统将提示你输入新的UNIX密码。输入你想要设置的新密码并按回车键。\n\n6. 系统将要求你重新输入新密码以进行确认。再次输入新密码并按回车键。\n\n7. 如果密码更改成功，系统将显示“密码已成功更新”的消息。\n\n8. 使用`exit`命令断开与服务器的连接。\n\n现在你已经成功更改了root账户的密码。下次使用SSH连接到服务器时，请使用新的密码进行身份验证。"}]}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "PHP算法逻辑：如何计算年龄？"}, {"role": "assistant", "content": "要计算年龄，您可以使用以下PHP算法。此算法考虑了生日、当前日期以及闰年的计算。\n\n```php\nfunction calculateAge($birthdate) {\n  // 格式为YYYY-MM-DD的生日\n  $birthdate = strtotime($birthdate);\n  \n  // 获取当前日期\n  $now = time();\n\n  // 计算年龄\n  $age = date('Y', $now) - date('Y', $birthdate);\n  \n  // 如果还没有过生日，减去一年\n  if (date('m-d', $now) < date('m-d', $birthdate)) {\n    $age--;\n  }\n\n  return $age;\n}\n\n// 使用\n$birthdate = \"2000-01-01\";\necho \"年龄是: \" . calculateAge($birthdate) . \" 岁\";\n```\n\n这个函数将生日作为输入，输出用户的年龄。首先，它将生日转换为时间戳，然后获取当前的时间戳。接下来，它计算年份之间的差异。如果当前日期仍未超过这一年的生日，这个函数将年龄减少1。\n\n在使用函数时，只需传入出生日期即可，函数会返回相应的年龄。"}]}
+{"messages": [{"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "给我解释一下什么是胰岛素抵抗？ 请详细描述胰岛素抵抗的症状、原因与治疗方法。 胰岛素抵抗会影响一个人的日常生活吗？ 有哪些生活习惯改变可以帮助改善胰岛素抵抗？"}, {"role": "assistant", "content": "胰岛素抵抗是指身体对胰岛素的敏感性降低，胰岛素不能有效地发挥作用，导致血糖无法正常进入身体细胞。胰岛素是一种激素，由胰腺产生，其主要作用是帮助身体细胞吸收血糖并将其转化为能量。当胰岛素抵抗发生时，胰腺会分泌更多的胰岛素以维持正常的血糖水平，但长期的胰岛素抵抗可能导致2型糖尿病和其他健康问题的发生。\n\n胰岛素抵抗的症状可能包括：\n\n1. 高血糖：由于胰岛素不能正常工作，血糖无法进入细胞，导致血糖升高。\n2. 疲劳：由于身体无法有效利用血糖产生能量，患者可能会感到疲劳无力。\n3. 肥胖：胰岛素抵抗可能导致过量的胰岛素分泌，使得脂肪存储增加，进而导致体重增加。\n4. 高血压：胰岛素抵抗与高血压之间存在关联，可能导致血压升高。\n5. 黑棘皮病：皮肤出现黑色、粗糙的皮疹，通常出现在颈部、腋窝等皮肤皱褶处。\n\n胰岛素抵抗的原因可能包括：\n\n1. 遗传因素：遗传易感基因可能会增加一个人患胰岛素抵抗的风险。\n2. 肥胖：过重或肥胖会增加患胰岛素抵抗的风险。\n3. 缺乏运动：缺乏运动可能会导致肌肉对胰岛素的敏感性降低。\n4. 高糖饮食：长期摄入高糖食品可能导致胰岛素抵抗。\n5. 睡眠不足：长期睡眠不足可能会影响胰岛素的正常功能。\n\n治疗胰岛素抵抗的方法主要包括生活方式的改变和药物治疗：\n\n1. 减轻体重：通过改善饮食和增加运动来减轻体重，有助于提高身体对胰岛素的敏感性。\n2. 增加运动：增加有氧运动和力量训练，有助于提高肌肉对胰岛素的敏感性。\n3. 改善饮食：摄入低糖、高纤维、富含全谷物和蔬菜水果的食物，有助于控制血糖和减轻体重。\n4. 药物治疗：在必要时，医生可能会开处降糖药物，如二甲双胍，以帮助提高胰岛素敏感性。\n\n胰岛素抵抗会影响一个人的日常生活，包括疲劳、体重增加、血糖波动等问题。通过改变生活习惯，如减轻体重、增加运动、改善饮食和保证充足睡眠，可以帮助改善胰岛素抵抗，降低患2型糖尿病的风险。"}]}

finetune/deepspeed_support/ds_zero2_no_offload.json

@@ -0,0 +1,28 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": "auto"
+    },
+    "zero_optimization": {
+        "stage": 2,
+        "allgather_partitions": true,
+        "allgather_bucket_size": 1e8,
+        "overlap_comm": true,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 1e8,
+        "contiguous_gradients": true
+    },
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 10,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}

finetune/deepspeed_support/ds_zero2_offload.json

@@ -0,0 +1,34 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": "auto"
+    },
+
+    "zero_optimization": {
+        "stage": 2,
+        "offload_optimizer": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "allgather_partitions": true,
+        "allgather_bucket_size": 5e8,
+        "overlap_comm": true,
+        "reduce_scatter": true,
+        "reduce_bucket_size": 5e8,
+        "contiguous_gradients": true
+    },
+
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 10,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}

finetune/deepspeed_support/ds_zero3_no_offload.json

@@ -0,0 +1,41 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": "auto"
+    },
+
+    "zero_optimization": {
+        "stage": 3,
+        "offload_optimizer": {
+            "device": "none",
+            "pin_memory": true
+        },
+        "offload_param": {
+            "device": "none",
+            "pin_memory": true
+        },
+        "overlap_comm": true,
+        "contiguous_gradients": true,
+        "sub_group_size": 1e9,
+        "reduce_bucket_size": "auto",
+        "stage3_prefetch_bucket_size": "auto",
+        "stage3_param_persistence_threshold": "auto",
+        "stage3_max_live_parameters": 1e9,
+        "stage3_max_reuse_distance": 1e9,
+        "stage3_gather_16bit_weights_on_model_save": false
+    },
+
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 10,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}

finetune/deepspeed_support/ds_zero3_offload.json

@@ -0,0 +1,41 @@
+{
+    "fp16": {
+        "enabled": "auto",
+        "loss_scale": 0,
+        "loss_scale_window": 1000,
+        "initial_scale_power": 16,
+        "hysteresis": 2,
+        "min_loss_scale": 1
+    },
+    "bf16": {
+        "enabled": "auto"
+    },
+
+    "zero_optimization": {
+        "stage": 3,
+        "offload_optimizer": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "offload_param": {
+            "device": "cpu",
+            "pin_memory": true
+        },
+        "overlap_comm": true,
+        "contiguous_gradients": true,
+        "sub_group_size": 1e9,
+        "reduce_bucket_size": "auto",
+        "stage3_prefetch_bucket_size": "auto",
+        "stage3_param_persistence_threshold": "auto",
+        "stage3_max_live_parameters": 1e9,
+        "stage3_max_reuse_distance": 1e9,
+        "stage3_gather_16bit_weights_on_model_save": false
+    },
+
+    "gradient_accumulation_steps": "auto",
+    "gradient_clipping": "auto",
+    "steps_per_print": 10,
+    "train_batch_size": "auto",
+    "train_micro_batch_size_per_gpu": "auto",
+    "wall_clock_breakdown": false
+}

finetune/deepspeed_support/train.py

@@ -0,0 +1,603 @@
+# Copyright 2024 Tencent Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
+#
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
+# and OPT implementations in this library. It has been modified from its
+# original forms to accommodate minor architectural differences compared
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+import json
+import torch
+import shutil
+import logging
+from dataclasses import dataclass, field
+from typing import Optional, Dict
+
+import transformers
+from torch.utils.data import Dataset
+from transformers import Trainer, TrainerCallback
+from peft import LoraConfig, get_peft_model, PeftModel
+from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
+from transformers.modeling_utils import unwrap_model
+
+
+def print_args(args, name='arguments'):
+    """Print arguments."""
+    if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0:
+        print(f'------------------------ {name} ------------------------', flush=True)
+        str_list = []
+        for arg in vars(args):
+            dots = '.' * (48 - len(arg))
+            str_list.append('  {} {} {}'.format(arg, dots, getattr(args, arg)))
+        for arg in sorted(str_list, key=lambda x: x.lower()):
+            print(arg, flush=True)
+        print(f'-------------------- end of {name} ---------------------', flush=True)
+
+
+@dataclass
+class ModelArguments:
+    use_flash_attn: bool = field(
+        default=False, 
+        metadata={"help": "Enable FlashAttention-2 for faster training."}
+    )
+    use_lora: bool = field(default=False, metadata={"help": "Enable Lora for faster training."})
+    lora_rank: int = field(default=64, metadata={"help": "The rank of lora."})
+    lora_alpha: int = field(default=8, metadata={"help": "Lora alpha"})
+    lora_dropout: float = field(default=0.0, metadata={"help": "Lora dropout"})
+
+
+@dataclass
+class DataArguments:
+    train_data_file: str = field(default=None, metadata={"help": "Path to the training data."})
+    max_seq_length: int = field(
+        default=2048, 
+        metadata={"help": "The max sequence length of the model inputs after tokenization."}
+    )
+    complex_data: Optional[str] = field(default=None)
+    use_dummy_data: bool = field(default=False, metadata={"help": "Use dummy data."})
+
+
+@dataclass
+class TrainingArguments(transformers.TrainingArguments):
+    cache_dir: Optional[str] = field(default=None)
+    optim: str = field(default="adamw_torch")
+    tokenizer_name_or_path: Optional[str] = field(default=None)
+    model_name_or_path: Optional[str] = field(default=None)
+    min_lr: float = field(
+        default=0.01, 
+        metadata={"help": "The final learning rate at the end of the decay will be learning_rate * min_lr"}
+    )
+
+
+IGNORE_INDEX = -100
+
+HY_START_ID = 120000
+HY_MIDDLE_ID = 120001
+HY_END_ID = 120025
+
+
+class DummyDataset(Dataset):
+    def __init__(self, tokenizer, max_seq_length=512, length=1000):
+        self.tokenizer = tokenizer
+        self.max_seq_length = max_seq_length
+        self.length = length
+    
+    def __len__(self):
+        return self.length
+    
+    def __getitem__(self, index):
+        tokens = torch.randint(0, self.tokenizer.vocab_size, (self.max_seq_length, ))
+        return {'input_ids': tokens, 'labels': tokens}
+
+
+class SFTDataset(Dataset):
+    def __init__(self, data_file, tokenizer, max_seq_length = 2048, prompt_format = 'mplus'):
+        self.tokenizer = tokenizer
+        self.prompt_format = prompt_format
+        self.max_seq_length = max_seq_length
+
+        self.data_list = self.load_data(data_file)
+
+        # Pre-compute special token IDs for loss masking (Scheme B)
+        self.hy_start_id = HY_START_ID
+        self.hy_middle_id = HY_MIDDLE_ID
+        self.hy_end_id = HY_END_ID
+        # "assistant" is encoded as BPE tokens [611, 10372] ("ass" + "istant")
+        self.assistant_bpe_ids = tokenizer.encode('assistant', add_special_tokens=False)
+        self.pad_token_id = tokenizer.pad_token_id
+
+    def __len__(self):
+        return len(self.data_list)
+
+    def load_data(self, data_file):
+        logging.info('Loading data: {}'.format(data_file))
+        with open(data_file, 'r', encoding='utf8') as f:
+            data_list = f.readlines()
+        logging.info("there are {} data in dataset".format(len(data_list)))
+        return data_list
+
+    def _find_assistant_turn_boundaries(self, token_ids):
+        """Find assistant turn boundaries using Scheme B.
+        
+        Locates assistant turns by finding <hy_start> + "assistant" BPE tokens pattern,
+        then marks loss from the <hy_middle> token (exclusive) to <hy_end> (inclusive).
+        
+        Returns:
+            List of (start, end) tuples where:
+            - start: position of <hy_middle> in the assistant turn (loss starts at start+1)
+            - end: position of <hy_end> for that turn (loss includes this position)
+        """
+        boundaries = []
+        assistant_len = len(self.assistant_bpe_ids)
+        ids_list = token_ids.tolist() if isinstance(token_ids, torch.Tensor) else token_ids
+        n = len(ids_list)
+        
+        i = 0
+        while i < n:
+            # Look for <hy_start> token
+            if ids_list[i] == self.hy_start_id:
+                # Check if followed by "assistant" BPE tokens
+                role_start = i + 1
+                role_end = role_start + assistant_len
+                if role_end <= n and ids_list[role_start:role_end] == self.assistant_bpe_ids:
+                    # Check if followed by <hy_middle>
+                    if role_end < n and ids_list[role_end] == self.hy_middle_id:
+                        middle_pos = role_end
+                        # Find the corresponding <hy_end>
+                        end_pos = None
+                        for j in range(middle_pos + 1, n):
+                            if ids_list[j] == self.hy_end_id:
+                                end_pos = j
+                                break
+                        if end_pos is not None:
+                            boundaries.append((middle_pos, end_pos))
+                            i = end_pos + 1
+                            continue
+            i += 1
+        
+        return boundaries
+
+    def encode_data(self, data_dict):
+        model_inputs = {}
+        reasoning_effort = data_dict.get('reasoning_effort', None)
+        if reasoning_effort is None:
+            reasoning_effort = 'no_think'
+        try:
+            template_output = self.tokenizer.apply_chat_template(
+                data_dict['messages'], tokenize=True, return_dict=False,
+                reasoning_effort=reasoning_effort
+            )
+        except Exception as e:
+            print(f"[ERROR] apply_chat_template failed: {e}")
+            print(f"[ERROR] messages: {data_dict['messages']}")
+            print(f"[ERROR] reasoning_effort: {reasoning_effort}")
+            template_output = []
+        
+        # Debug: Check template_output type and content
+        if isinstance(template_output, bool):
+            print(f"[WARNING] apply_chat_template returned bool: {template_output}")
+            print(f"[WARNING] messages: {data_dict['messages']}")
+            print(f"[WARNING] reasoning_effort: {reasoning_effort}")
+            template_output = []
+        
+        if isinstance(template_output, list) and len(template_output) > 0 and isinstance(template_output[0], list):
+            template_output = template_output[0]
+        
+        # Ensure template_output is a list of integers
+        if not isinstance(template_output, list) or not all(isinstance(x, int) for x in template_output):
+            print(f"[WARNING] Invalid template_output format: {type(template_output)}, content: {template_output}")
+            print(f"[WARNING] messages: {data_dict['messages']}")
+            template_output = []
+        
+        message_tokens = torch.tensor(template_output, dtype=torch.long)
+
+        # Handle empty message_tokens case
+        if message_tokens.numel() == 0:
+            print(f"[WARNING] Empty message_tokens, skipping data sample")
+            input_ids = torch.tensor([], dtype=torch.long)
+            labels = torch.tensor([], dtype=torch.long)
+            attention_mask = torch.tensor([], dtype=torch.bool)
+        else:
+            # Scheme B: Find assistant turn boundaries and build labels
+            boundaries = self._find_assistant_turn_boundaries(message_tokens)
+            message_labels = torch.full_like(message_tokens, IGNORE_INDEX)
+            
+            for middle_pos, end_pos in boundaries:
+                # Compute loss from the token after <hy_middle> to <hy_end> (inclusive)
+                message_labels[middle_pos + 1:end_pos + 1] = message_tokens[middle_pos + 1:end_pos + 1]
+            
+            input_ids = message_tokens.to(torch.long)
+            labels = message_labels.to(torch.long)
+
+            input_ids = input_ids[:self.max_seq_length]
+            labels = labels[:self.max_seq_length]
+            attention_mask = input_ids.ne(self.pad_token_id).to(torch.bool)
+
+        model_inputs["input_ids"] = input_ids
+        model_inputs["attention_mask"] = attention_mask
+        model_inputs["labels"] = labels
+
+        return model_inputs
+
+    def __getitem__(self, index):
+        data = self.data_list[index]
+        data = json.loads(data)
+        model_inputs = self.encode_data(data)
+        
+        # Check if the encoded data is empty (due to tokenization failure)
+        if model_inputs["input_ids"].numel() == 0:
+            # Return a valid placeholder sample to avoid crash
+            eos_token_id = self.hy_end_id
+            pad_token_id = self.pad_token_id
+            
+            # Create a minimal valid sequence
+            placeholder_tokens = [self.hy_start_id, eos_token_id]
+            placeholder_tokens = placeholder_tokens[:self.max_seq_length]
+            
+            input_ids = torch.tensor(placeholder_tokens, dtype=torch.long)
+            labels = torch.tensor([IGNORE_INDEX, IGNORE_INDEX], dtype=torch.long)[:self.max_seq_length]
+            attention_mask = torch.tensor([1, 1], dtype=torch.bool)[:self.max_seq_length]
+            
+            # Pad to max_seq_length if needed
+            if len(placeholder_tokens) < self.max_seq_length:
+                padding_length = self.max_seq_length - len(placeholder_tokens)
+                input_ids = torch.cat([input_ids, torch.full((padding_length,), pad_token_id, dtype=torch.long)])
+                labels = torch.cat([labels, torch.full((padding_length,), IGNORE_INDEX, dtype=torch.long)])
+                attention_mask = torch.cat([attention_mask, torch.zeros(padding_length, dtype=torch.bool)])
+            
+            model_inputs = {
+                "input_ids": input_ids,
+                "attention_mask": attention_mask,
+                "labels": labels
+            }
+
+        return model_inputs
+
+
+@dataclass
+class DataCollatorForSupervisedDataset(object):
+    """Collate examples for supervised fine-tuning."""
+
+    tokenizer: transformers.PreTrainedTokenizer
+
+    def __call__(self, instances):
+        input_ids = [instance['input_ids'] for instance in instances]
+        labels = [instance['labels'] for instance in instances]
+        pad_token_id = self.tokenizer.pad_token_id
+        input_ids = torch.nn.utils.rnn.pad_sequence(input_ids, batch_first=True, padding_value=pad_token_id)
+        labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=IGNORE_INDEX)
+        return dict(
+            input_ids=input_ids,
+            labels=labels,
+            attention_mask=input_ids.ne(pad_token_id),
+        )
+
+
+def make_supervised_data_module(tokenizer, data_args) -> Dict:
+    """Make dataset and collator for supervised fine-tuning."""
+    if data_args.use_dummy_data:
+        train_dataset = DummyDataset(tokenizer, data_args.max_seq_length)
+    else:
+        train_dataset = SFTDataset(
+            tokenizer=tokenizer, 
+            data_file=data_args.train_data_file, 
+            max_seq_length=data_args.max_seq_length
+        )
+    data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer)
+    return dict(train_dataset=train_dataset, eval_dataset=None, data_collator=data_collator)
+
+
+# Copy tokenizer and config files when saving checkpoints
+class CustomSaveCallback(TrainerCallback):
+    def on_save(self, args, state, control, **kwargs):
+        if not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0:
+            output_dir = os.path.join(args.output_dir, f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}")
+
+            # Copy tokenizer and config files to checkpoint directory
+            tokenizer_files = [
+                'config.json',
+                'generation_config.json',
+                'tokenizer_config.json',
+                'tokenizer.json',
+                'chat_template.jinja',
+                'preprocessor_config.json',
+                'hy.tiktoken',
+                'tokenization_hy.py',
+                'special_tokens_map.json',
+            ]
+            for fname in tokenizer_files:
+                src = os.path.join(args.tokenizer_name_or_path, fname)
+                if os.path.isfile(src):
+                    shutil.copy(src, os.path.join(output_dir, fname))
+
+        return control
+
+
+def train():
+    parser = transformers.HfArgumentParser((ModelArguments, DataArguments, TrainingArguments))
+    model_args, data_args, training_args = parser.parse_args_into_dataclasses()
+    print_args(model_args, 'model arguments')
+    print_args(data_args, 'data arguments')
+    print_args(training_args, 'training arguments')
+
+    tokenizer = transformers.AutoTokenizer.from_pretrained(
+        training_args.tokenizer_name_or_path,
+        trust_remote_code = True
+    )
+
+    init_kwargs = {}
+    if model_args.use_flash_attn:
+        init_kwargs["attn_implementation"] = "flash_attention_2"
+        # Workaround: transformers >= 5.x uses importlib.metadata.packages_distributions()
+        # to verify flash-attn package name, which fails when the package is installed under
+        # a custom distribution name (e.g. ptm-flash-attn). Patch the check to skip it.
+        try:
+            from transformers.modeling_flash_attention_utils import FLASH_ATTENTION_COMPATIBILITY_MATRIX
+            _orig_pkg_check = FLASH_ATTENTION_COMPATIBILITY_MATRIX[2]["pkg_availability_check"]
+            FLASH_ATTENTION_COMPATIBILITY_MATRIX[2]["pkg_availability_check"] = lambda *a, **kw: True
+            print("[Patch] Bypassed flash_attn package distribution name check for FA2.")
+        except Exception as e:
+            print(f"[Patch] Could not patch FA2 pkg check (non-fatal): {e}")
+
+    # Determine torch dtype
+    if training_args.bf16:
+        torch_dtype = torch.bfloat16
+    elif training_args.fp16:
+        torch_dtype = torch.float16
+    else:
+        torch_dtype = torch.float32
+
+    # -----------------------------------------------------------------------
+    # DeepSpeed ZeRO-3: Tell transformers that we are using ZeRO-3 so that
+    # from_pretrained will shard the model across ranks instead of loading
+    # the full model on each node's CPU (which would OOM for large models).
+    #
+    # NOTE: The ds_config may contain "auto" for batch size fields, which
+    # DeepSpeed cannot parse at this stage (before Trainer resolves them).
+    # We must fill in concrete values before passing to HfDeepSpeedConfig.
+    # -----------------------------------------------------------------------
+    if training_args.deepspeed:
+        from transformers.integrations import HfDeepSpeedConfig
+
+        # Load ds_config and resolve "auto" batch size fields
+        ds_config = training_args.deepspeed
+        if isinstance(ds_config, str):
+            with open(ds_config, 'r') as f:
+                ds_config = json.load(f)
+
+        # Fill in batch size fields that DeepSpeed needs for zero.Init
+        if ds_config.get("train_micro_batch_size_per_gpu", "auto") == "auto":
+            ds_config["train_micro_batch_size_per_gpu"] = training_args.per_device_train_batch_size
+        if ds_config.get("gradient_accumulation_steps", "auto") == "auto":
+            ds_config["gradient_accumulation_steps"] = training_args.gradient_accumulation_steps
+        if ds_config.get("train_batch_size", "auto") == "auto":
+            ds_config["train_batch_size"] = (
+                training_args.per_device_train_batch_size
+                * training_args.gradient_accumulation_steps
+                * (torch.distributed.get_world_size() if torch.distributed.is_initialized() else 1)
+            )
+
+        dschf = HfDeepSpeedConfig(ds_config)  # noqa: F841 - must keep ref to avoid GC
+
+    # Check if model weights exist (not just the directory)
+    _has_weights = (
+        training_args.model_name_or_path is not None
+        and os.path.isdir(training_args.model_name_or_path)
+        and any(
+            os.path.isfile(os.path.join(training_args.model_name_or_path, f))
+            for f in ("model.safetensors", "pytorch_model.bin", "model.safetensors.index.json", "pytorch_model.bin.index.json")
+        )
+    )
+
+    if _has_weights:
+        print(f"Loading model from: {training_args.model_name_or_path}")
+        model = transformers.AutoModelForCausalLM.from_pretrained(
+            training_args.model_name_or_path,
+            trust_remote_code=True,
+            dtype=torch_dtype,
+            attn_implementation=init_kwargs.get("attn_implementation", None),
+        )
+        print(f"[HY4] Model loaded successfully via from_pretrained.")
+    else:
+        if training_args.model_name_or_path is None:
+            raise ValueError(
+                "--model_name_or_path must be specified. Cannot load model config from None. "
+                "Please provide the path to the model directory."
+            )
+        print(f"Model weights not found at: {training_args.model_name_or_path}, "
+              f"using random initialized model instead.")
+        config = transformers.AutoConfig.from_pretrained(
+            training_args.model_name_or_path,
+            trust_remote_code=True
+        )
+        model = transformers.AutoModelForCausalLM.from_config(
+            config,
+            trust_remote_code=True,
+            dtype=torch_dtype,
+            attn_implementation=init_kwargs.get("attn_implementation", None),
+        )
+    
+    if model_args.use_lora:
+        # HY4 uses MLA (Multi-head Latent Attention) with different projection names
+        lora_config = LoraConfig(
+            r=model_args.lora_rank,
+            lora_alpha=model_args.lora_alpha,
+            lora_dropout=model_args.lora_dropout,
+            target_modules=["q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj", "o_proj"],
+            bias="none",
+            task_type="CAUSAL_LM",
+        )
+        model = get_peft_model(model, lora_config)
+
+        # Fix: Mark PEFT LoRA wrapper modules as ZeRO-3 leaf modules.
+        # PEFT wraps target Linear layers with lora.Linear, adding extra
+        # sub-modules (base_layer, lora_A, lora_B). This changes the module
+        # tree structure and disrupts ZeRO-3's parameter fetch/release
+        # scheduling, causing OOM during backward recomputation.
+        # By marking these wrappers as z3_leaf, ZeRO-3 treats them as atomic
+        # units (same as the original Linear), restoring correct scheduling.
+        from deepspeed.utils import set_z3_leaf_module
+        from peft.tuners.lora import Linear as LoraLinear
+        z3_leaf_count = 0
+        for module in model.modules():
+            if isinstance(module, LoraLinear):
+                set_z3_leaf_module(module, True)
+                z3_leaf_count += 1
+        print(f"[z3_leaf] Marked {z3_leaf_count} LoraLinear modules with _z3_leaf=True", flush=True)
+
+        # Verify the attribute is actually set
+        verified_count = 0
+        for name, module in model.named_modules():
+            if isinstance(module, LoraLinear):
+                has_attr = getattr(module, '_z3_leaf', False)
+                if has_attr:
+                    verified_count += 1
+                else:
+                    print(f"[z3_leaf] WARNING: module '{name}' is LoraLinear but _z3_leaf={has_attr}", flush=True)
+        print(f"[z3_leaf] Verification after marking: {verified_count}/{z3_leaf_count} modules have _z3_leaf=True", flush=True)
+
+    data_module = make_supervised_data_module(tokenizer=tokenizer, data_args=data_args)
+    # Tell Trainer not to attempt DataParallel
+    model.is_parallelizable = True
+    model.model_parallel = True
+
+    training_args.lr_scheduler_kwargs = {
+        'min_lr_rate': training_args.min_lr / training_args.learning_rate,
+    }
+
+    # -----------------------------------------------------------------------
+    # Fix: DeepSpeed ZeRO-3 + gradient checkpointing compatibility.
+    #
+    # PyTorch's torch.utils.checkpoint with use_reentrant=False (the default
+    # in transformers) performs strict metadata checks on recomputed tensors
+    # during backward.  Under ZeRO-3, parameters are all-gathered during the
+    # first forward pass (shape=[full_size]) but may be partitioned back
+    # (shape=[0]) when the checkpoint recomputes, causing a CheckpointError.
+    #
+    # Setting use_reentrant=True avoids this strict metadata check.
+    # -----------------------------------------------------------------------
+    if training_args.gradient_checkpointing and training_args.deepspeed:
+        training_args.gradient_checkpointing_kwargs = {"use_reentrant": True}
+
+    trainer = Trainer(
+        model=model, 
+        processing_class=tokenizer, 
+        args=training_args,
+        callbacks=[CustomSaveCallback],
+        **data_module
+    )
+    model.config.use_cache = False
+
+    # -----------------------------------------------------------------------
+    # Monkey-patch: fix dtype mismatch in DeepSpeed ZeRO-3 linear wrapper.
+    #
+    # By this point the DeepSpeed engine has been initialised by the Trainer
+    # and torch.nn.functional.linear has been replaced with
+    # zero3_linear_wrap.  That wrapper does NOT auto-align input/weight
+    # dtypes before the matmul, causing "expected mat1 and mat2 to have the
+    # same dtype" errors in mixed-precision paths (e.g. enable_lm_head_fp32
+    # casts input to fp32 but weight remains bf16 under ZeRO-3).
+    #
+    # We wrap F.linear HERE (after DeepSpeed init) so that:
+    #   1. We are sure to capture the already-replaced function.
+    #   2. The dtype cast happens *outside* the autograd.Function, so
+    #      gradient-checkpointing recompute sees identical tensor metadata.
+    # -----------------------------------------------------------------------
+    import torch.nn.functional as _F
+    _orig_F_linear = _F.linear
+
+    def _dtype_safe_linear(input, weight, bias=None):
+        if input.dtype != weight.dtype:
+            input = input.to(weight.dtype)
+        return _orig_F_linear(input, weight, bias)
+
+    _F.linear = _dtype_safe_linear
+    # -----------------------------------------------------------------------
+
+    # -----------------------------------------------------------------------
+    # Monkey-patch: skip grad norm calculation when max_grad_norm == 0.
+    #
+    # Under ZeRO-3 + CPU offload, DeepSpeed's complete_grad_norm_calculation
+    # all-gathers every gradient on CPU and does an ALLREDUCE to compute the
+# global L2 norm. For a large model this is extremely slow and triggers an
+    # NCCL ALLREDUCE timeout (NumelIn=1) at optimizer step, even when clipping
+    # is disabled via gradient_clipping=0.0 (which only skips the clip, not the
+    # computation).
+    #
+    # When the user explicitly sets --max_grad_norm 0 we fully skip the norm
+    # computation (no all-gather, no ALLREDUCE) by returning 0.0 early.
+    #
+    # NOTE: We patch the CLASS method (not instance) because trainer.deepspeed
+    # is None at this point — the DeepSpeed engine is created inside
+    # trainer.train(). By patching the class, any future engine instance will
+    # inherit the patched method.
+    # -----------------------------------------------------------------------
+    if getattr(training_args, "max_grad_norm", None) == 0:
+        import torch as _torch
+        from deepspeed.runtime.zero.stage3 import DeepSpeedZeroOptimizer_Stage3 as _ZeRO3Optimizer
+
+        def _skip_get_norm_groups(self):
+            return [_torch.tensor(0.0)]
+
+        _ZeRO3Optimizer._get_norm_groups = _skip_get_norm_groups
+        logging.info("[grad_norm] max_grad_norm=0: patched DeepSpeedZeroOptimizer_Stage3._get_norm_groups to skip norm computation")
+    # -----------------------------------------------------------------------
+
+    # -----------------------------------------------------------------------
+    # Post-DeepSpeed-init verification: check if _z3_leaf marks survived
+    # DeepSpeed engine initialization (which happens inside Trainer.__init__).
+    # -----------------------------------------------------------------------
+    if model_args.use_lora:
+        from peft.tuners.lora import Linear as LoraLinear
+        post_init_count = 0
+        post_init_verified = 0
+        for name, module in trainer.model.named_modules():
+            if isinstance(module, LoraLinear):
+                post_init_count += 1
+                has_attr = getattr(module, '_z3_leaf', False)
+                if has_attr:
+                    post_init_verified += 1
+                elif post_init_count <= 5:  # Only print first few warnings to avoid spam
+                    print(f"[z3_leaf] POST-INIT WARNING: module '{name}' lost _z3_leaf after Trainer init!", flush=True)
+        print(f"[z3_leaf] Post-Trainer-init verification: {post_init_verified}/{post_init_count} LoraLinear modules still have _z3_leaf=True", flush=True)
+    # -----------------------------------------------------------------------
+
+    trainer.train(resume_from_checkpoint=training_args.resume_from_checkpoint)
+
+    # Synchronize all processes before exit to avoid "Connection reset by peer"
+    # warnings caused by timing differences in multi-node shutdown.
+    if torch.distributed.is_initialized():
+        torch.distributed.barrier()
+
+
+if __name__ == "__main__":
+    train()

finetune/deepspeed_support/train.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+
+NET_TYPE="high"
+export NCCL_DEBUG=WARN
+export NCCL_P2P_LEVEL=NVL
+export NCCL_IB_TIMEOUT=24
+export NCCL_NVLS_ENABLE=0
+export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
+if [[ "${NET_TYPE}" = "low" ]]; then
+    export NCCL_SOCKET_IFNAME=eth1
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_HCA=mlx5_2:1
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+else
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_IB_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+    export NCCL_SOCKET_IFNAME=bond1
+    export UCX_NET_DEVICES=bond1
+    export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
+    export NCCL_COLLNET_ENABLE=0
+    export SHARP_COLL_ENABLE_SAT=0
+    export NCCL_NET_GDR_LEVEL=2
+    export NCCL_IB_QPS_PER_CONNECTION=4
+    export NCCL_IB_TC=160
+    export NCCL_PXN_DISABLE=1
+fi
+
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
+IP_LIST=${IP_LIST:-"127.0.0.1"}
+
+IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
+export NODES=${#IP_ARRAY[@]}
+export LOCAL_IP=${IP_ARRAY[0]}
+NODE_IP_LIST=""
+for ip in "${IP_ARRAY[@]}"; do
+    if [ -n "$NODE_IP_LIST" ]; then
+        NODE_IP_LIST="${NODE_IP_LIST},"
+    fi
+    NODE_IP_LIST="${NODE_IP_LIST}${ip}:${HOST_GPU_NUM}"
+done
+export NODE_IP_LIST
+export NODE_NUM=$((${NODES} * ${HOST_GPU_NUM}))
+
+model_path=/path/to/hy_ckpt
+tokenizer_path=/path/to/hy_ckpt
+train_data_file=../data/example_data.jsonl
+
+ds_config_file=ds_zero3_offload.json
+
+output_path=./output
+
+mkdir -p ${output_path}
+
+current_time=$(date "+%Y.%m.%d-%H.%M.%S")
+log_file=${output_path}/"log_${current_time}.txt"
+
+echo $NODE_IP_LIST > env.txt 2>&1
+sed "s/:/ slots=/g" env.txt | sed "s/,/\n/g" >  "hostfile"
+sed "s/:.//g" env.txt | sed "s/,/\n/g" >  "pssh.hosts"
+export CHIEF_IP=$LOCAL_IP
+
+if [ ${NODES} -gt 1 ]; then
+    HOST_PATH=hostfile
+    DS_ARGS="--hostfile=${HOST_PATH} --master_addr ${CHIEF_IP}"
+else
+    DS_ARGS=""
+fi
+
+echo "NODES: ${NODES}, LOCAL_IP: ${LOCAL_IP}, NODE_IP_LIST: ${NODE_IP_LIST}"
+
+deepspeed ${DS_ARGS} \
+    train.py \
+    --do_train \
+    --model_name_or_path ${model_path} \
+    --tokenizer_name_or_path ${tokenizer_path} \
+    --train_data_file ${train_data_file} \
+    --deepspeed ${ds_config_file} \
+    --output_dir ${output_path} \
+    --per_device_train_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --gradient_checkpointing \
+    --lr_scheduler_type cosine_with_min_lr \
+    --logging_steps 1 \
+    --save_steps 20 \
+    --learning_rate 1e-5 \
+    --min_lr 1e-6 \
+    --warmup_steps 0 \
+    --save_strategy steps \
+    --bf16 \
+    --max_grad_norm 0 \
+    --max_seq_length 1024 | tee ${log_file}

finetune/deepspeed_support/train_lora.sh

@@ -0,0 +1,109 @@
+#!/bin/bash
+
+NET_TYPE="high"
+export NCCL_DEBUG=WARN
+export NCCL_P2P_LEVEL=NVL
+export NCCL_IB_TIMEOUT=24
+export NCCL_NVLS_ENABLE=0
+export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+if [[ "${NET_TYPE}" = "low" ]]; then
+    export NCCL_SOCKET_IFNAME=eth1
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_HCA=mlx5_2:1
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+else
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_IB_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+    export NCCL_SOCKET_IFNAME=bond1
+    export UCX_NET_DEVICES=bond1
+    export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
+    export NCCL_COLLNET_ENABLE=0
+    export SHARP_COLL_ENABLE_SAT=0
+    export NCCL_NET_GDR_LEVEL=2
+    export NCCL_IB_QPS_PER_CONNECTION=4
+    export NCCL_IB_TC=160
+    export NCCL_PXN_DISABLE=1
+fi
+
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "192.168.1.1,192.168.1.2" or single node "192.168.1.1"
+IP_LIST=${IP_LIST:-"127.0.0.1"}
+
+IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
+export NODES=${#IP_ARRAY[@]}
+export LOCAL_IP=${IP_ARRAY[0]}
+NODE_IP_LIST=""
+for ip in "${IP_ARRAY[@]}"; do
+    if [ -n "$NODE_IP_LIST" ]; then
+        NODE_IP_LIST="${NODE_IP_LIST},"
+    fi
+    NODE_IP_LIST="${NODE_IP_LIST}${ip}:${HOST_GPU_NUM}"
+done
+export NODE_IP_LIST
+export NODE_NUM=$((${NODES} * ${HOST_GPU_NUM}))
+
+
+model_path=/path/to/hy_ckpt
+tokenizer_path=/path/to/hy_ckpt
+train_data_file=../data/example_data.jsonl
+
+ds_config_file=ds_zero3_offload.json
+
+output_path=./output
+
+mkdir -p ${output_path}
+
+current_time=$(date "+%Y.%m.%d-%H.%M.%S")
+log_file=${output_path}/"log_${current_time}.txt"
+
+echo $NODE_IP_LIST > env.txt 2>&1
+sed "s/:/ slots=/g" env.txt | sed "s/,/\n/g" >  "hostfile"
+sed "s/:.//g" env.txt | sed "s/,/\n/g" >  "pssh.hosts"
+export CHIEF_IP=$LOCAL_IP
+
+if [ ${NODES} -gt 1 ]; then
+    HOST_PATH=hostfile
+    DS_ARGS="--hostfile=${HOST_PATH} --master_addr ${CHIEF_IP}"
+else
+    DS_ARGS=""
+fi
+
+echo "NODES: ${NODES}, LOCAL_IP: ${LOCAL_IP}, NODE_IP_LIST: ${NODE_IP_LIST}"
+
+deepspeed ${DS_ARGS} \
+    train.py \
+    --do_train \
+    --model_name_or_path ${model_path} \
+    --tokenizer_name_or_path ${tokenizer_path} \
+    --train_data_file ${train_data_file} \
+    --deepspeed ${ds_config_file} \
+    --output_dir ${output_path} \
+    --per_device_train_batch_size 1 \
+    --gradient_accumulation_steps 1 \
+    --gradient_checkpointing \
+    --lr_scheduler_type cosine_with_min_lr \
+    --logging_steps 1 \
+    --save_steps 20 \
+    --learning_rate 1e-5 \
+    --min_lr 1e-6 \
+    --warmup_steps 0 \
+    --save_strategy steps \
+    --bf16 \
+    --max_grad_norm 0 \
+    --use_lora \
+    --lora_rank 64 \
+    --lora_alpha 128 \
+    --lora_dropout 0.1 \
+    --max_seq_length 512 | tee ${log_file}

finetune/llama_factory_support/dataset_info.json

@@ -0,0 +1,16 @@
+{
+  "hy_v4_demo": {
+    "file_name": "../data/example_data.jsonl",
+    "formatting": "sharegpt",
+    "columns": {
+      "messages": "messages"
+    },
+    "tags": {
+      "role_tag": "role",
+      "content_tag": "content",
+      "user_tag": "user",
+      "assistant_tag": "assistant",
+      "system_tag": "system"
+    }
+  }
+}

finetune/llama_factory_support/hy_v4_full_sft.yaml

@@ -0,0 +1,55 @@
+### model
+model_name_or_path: /path/to/hy_ckpt
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: full
+
+deepspeed: ../deepspeed_support/ds_zero3_offload.json
+
+### dataset
+dataset_dir: .
+dataset: hy_v4_demo
+template: hy_v4
+# NOTE: To train slow-thinking (CoT), include <think:opensource>...</think:opensource>
+# tags in assistant content. Without think tags, only fast-thinking is trained.
+cutoff_len: 256
+max_samples: 1000
+overwrite_cache: true
+
+### output
+output_dir: saves/hy_v4/full/sft
+logging_steps: 1
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 1.0e-5
+num_train_epochs: 3.0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1  # min_lr / learning_rate = 1e-6 / 1e-5 = 0.1
+warmup_steps: 0
+bf16: true
+optim: adamw_torch
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+ddp_timeout: 180000000
+flash_attn: auto
+max_grad_norm: 0.0
+resume_from_checkpoint: null
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

finetune/llama_factory_support/hy_v4_lora_sft.yaml

@@ -0,0 +1,71 @@
+### model
+model_name_or_path: /path/to/hy_ckpt
+trust_remote_code: true
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+
+
+
+# Using FSDP1 with full sharding strategy.
+fsdp: true
+fsdp_config:
+  version: 1
+  fsdp_sharding_strategy: FULL_SHARD
+  fsdp_backward_prefetch: BACKWARD_PRE
+  fsdp_state_dict_type: FULL_STATE_DICT
+  fsdp_use_orig_params: true
+  fsdp_cpu_ram_efficient_loading: true
+  fsdp_offload_params: false
+  fsdp_auto_wrap_policy: TRANSFORMER_BASED_WRAP
+  fsdp_transformer_layer_cls_to_wrap: HYV4DecoderLayer
+
+### LoRA parameters
+# HY4 uses MLA (Multi-head Latent Attention) with these projection names
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+lora_target: q_a_proj,q_b_proj,kv_a_proj_with_mqa,kv_b_proj,o_proj
+
+### dataset
+dataset_dir: .
+dataset: hy_v4_demo
+template: hy_v4
+cutoff_len: 1024
+max_samples: 1000
+overwrite_cache: true
+
+### output
+output_dir: saves/hy_v4/lora/sft
+logging_steps: 1
+save_steps: 500
+plot_loss: true
+overwrite_output_dir: true
+save_only_model: false
+report_to: none  # choices: [none, wandb, tensorboard, swanlab, mlflow]
+
+### train
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 2.0e-4
+num_train_epochs: 3.0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1  # min_lr / learning_rate = 2e-5 / 2e-4 = 0.1
+warmup_steps: 0
+bf16: true
+gradient_checkpointing: true
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+ddp_timeout: 180000000
+flash_attn: auto
+resume_from_checkpoint: null
+
+### eval
+# eval_dataset: alpaca_en_demo
+# val_size: 0.1
+# per_device_eval_batch_size: 1
+# eval_strategy: steps
+# eval_steps: 500

finetune/llama_factory_support/hy_v4_patches.py

@@ -0,0 +1,768 @@
+"""
+This module applies all necessary runtime patches.
+
+
+Patches applied:
+    1. ZeRO-3 key rename + expert fuse + buffer loading
+       Handles both inner format (original) and outer format (pre-converted) checkpoints.
+       - Key renaming: mlp.router.gate -> mlp.gate, etc.
+       - Per-expert -> 3D fuse: experts.N.gate_proj -> experts.gate_up_proj
+       - Buffer loading: e_score_correction_bias (ZeRO-3 only handles parameters)
+    2. Tokenizer file copy (CustomSaveCallback)
+       Ensures each checkpoint directory is self-contained for inference.
+"""
+
+import os
+import re
+import logging
+import shutil
+from typing import Optional
+
+import torch
+
+logger = logging.getLogger(__name__)
+
+# ============================================================================
+# Patch 1: Key rename + expert fuse + buffer loading for ZeRO-3
+#
+# The checkpoint may be in either inner format (original) or outer format
+# (pre-converted by convert_ckpt_to_outer.py). This patch handles both:
+#   - Key renaming: mlp.router.gate -> mlp.gate, etc.
+#   - Per-expert -> 3D fuse: experts.N.gate_proj -> experts.gate_up_proj
+#   - Buffer loading: e_score_correction_bias (ZeRO-3 only handles parameters)
+#
+# If the checkpoint is already in outer format, the rename/fuse logic is
+# effectively a no-op (no matching keys to transform).
+# ============================================================================
+
+# Key renames: checkpoint inner format -> model format
+_CKPT_KEY_RENAMES = [
+    ("mlp.router.gate.", "mlp.gate."),
+    ("mlp.expert_bias", "mlp.e_score_correction_bias"),
+    ("mlp.shared_mlp.", "mlp.shared_experts."),
+    # Also handle even older checkpoints that use mlp.gate.wg
+    ("mlp.gate.wg.", "mlp.gate."),
+]
+
+# Regex to match per-expert keys in checkpoint
+# e.g. model.layers.10.mlp.experts.5.gate_proj.weight
+_EXPERT_KEY_RE = re.compile(
+    r"^(.*\.mlp\.experts\.)(\d+)\.(gate_proj|up_proj|down_proj)\.weight$"
+)
+
+
+def _apply_buffer_loading_patch():
+    """Patch the DeepSpeed ZeRO-3 state_dict loader to handle:
+    1. Key renaming (inner checkpoint format -> model format)
+    2. Per-expert -> 3D fused expert tensors
+    3. Manual buffer loading (e_score_correction_bias etc.)
+
+    ZeRO-3's _load_state_dict_into_zero3_model only handles named_parameters.
+    Buffers like e_score_correction_bias must be loaded manually.
+    """
+    try:
+        from transformers.integrations.deepspeed import (
+            _load_state_dict_into_zero3_model as _orig_load_zero3,
+        )
+        import transformers.integrations.deepspeed as _ds_mod
+        import transformers.modeling_utils as _mu_mod
+    except ImportError:
+        logger.warning(
+            "Could not import transformers.integrations.deepspeed; "
+            "buffer loading patch NOT applied."
+        )
+        return
+
+    def _patched_load_zero3(model_to_load, state_dict, *args, **kwargs):
+        # Step 1: Key rename + per-expert collection
+        new_sd = {}
+        expert_groups = {}  # prefix -> {expert_idx -> {proj_name -> tensor}}
+
+        for k, v in state_dict.items():
+            m = _EXPERT_KEY_RE.match(k)
+            if m:
+                # Per-expert key: collect for fusion
+                prefix = m.group(1)
+                expert_idx = int(m.group(2))
+                proj_name = m.group(3)
+                if prefix not in expert_groups:
+                    expert_groups[prefix] = {}
+                if expert_idx not in expert_groups[prefix]:
+                    expert_groups[prefix][expert_idx] = {}
+                expert_groups[prefix][expert_idx][proj_name] = v
+            else:
+                # Non-expert key: apply simple renames
+                new_k = k
+                for old_sub, new_sub in _CKPT_KEY_RENAMES:
+                    if old_sub in new_k:
+                        new_k = new_k.replace(old_sub, new_sub)
+                new_sd[new_k] = v
+
+        # Step 2: Fuse expert groups into 3D tensors
+        if expert_groups:
+            for prefix in sorted(expert_groups.keys()):
+                experts = expert_groups[prefix]
+                num_experts = max(experts.keys()) + 1
+                gate_up_list = []
+                down_list = []
+                for i in range(num_experts):
+                    if i not in experts:
+                        logger.warning(
+                            "HYV4 Patch 1: Missing expert %d in %s", i, prefix
+                        )
+                        continue
+                    exp = experts[i]
+                    if "gate_proj" in exp and "up_proj" in exp:
+                        gate_up_list.append(
+                            torch.cat([exp["gate_proj"], exp["up_proj"]], dim=0)
+                        )
+                    if "down_proj" in exp:
+                        down_list.append(exp["down_proj"])
+                if gate_up_list:
+                    new_sd[f"{prefix}gate_up_proj"] = torch.stack(gate_up_list, dim=0)
+                if down_list:
+                    new_sd[f"{prefix}down_proj"] = torch.stack(down_list, dim=0)
+            logger.info(
+                "HYV4 Patch 1: Fused %d expert groups from per-expert to 3D format.",
+                len(expert_groups)
+            )
+            del expert_groups
+
+        # Step 3: Load parameters via original ZeRO-3 loader
+        result = _orig_load_zero3(model_to_load, new_sd, *args, **kwargs)
+
+        # Step 4: Manually load buffers (e.g. e_score_correction_bias)
+        # ZeRO-3's loader only handles named_parameters, not named_buffers.
+        buffers_loaded = 0
+        for name, buf in model_to_load.named_buffers():
+            if name in new_sd:
+                src_tensor = new_sd[name]
+                if isinstance(src_tensor, torch.Tensor):
+                    buf.data.copy_(src_tensor.to(buf.dtype))
+                    buffers_loaded += 1
+                    if isinstance(result, tuple) and len(result) >= 2:
+                        if isinstance(result[1], set):
+                            result[1].discard(name)
+        if buffers_loaded > 0:
+            logger.info(
+                "HYV4 Patch 1: Manually loaded %d buffers into model.",
+                buffers_loaded
+            )
+
+        del new_sd
+        return result
+
+    _ds_mod._load_state_dict_into_zero3_model = _patched_load_zero3
+    _mu_mod._load_state_dict_into_zero3_model = _patched_load_zero3
+    logger.info(
+        "HYV4 patch applied: ZeRO-3 key rename + expert fuse + buffer loading."
+    )
+
+# ============================================================================
+# Patch 2: Tokenizer file copy callback
+#
+# Ensures each checkpoint directory is self-contained for inference by
+# copying all tokenizer-related files from the original tokenizer path.
+# ============================================================================
+
+# Tokenizer files that should be copied to each checkpoint
+_TOKENIZER_FILES = [
+    "config.json",
+    "generation_config.json",
+    "tokenizer_config.json",
+    "tokenizer.json",
+    "chat_template.jinja",
+    "preprocessor_config.json",
+    "hy.tiktoken",
+    "tokenization_hy.py",
+    "special_tokens_map.json",
+]
+
+def _copy_tokenizer_to_checkpoint(tokenizer_dir: str, checkpoint_dir: str):
+    """Copy tokenizer files from tokenizer_dir to checkpoint_dir."""
+    for fname in _TOKENIZER_FILES:
+        src = os.path.join(tokenizer_dir, fname)
+        if os.path.isfile(src):
+            shutil.copy(src, os.path.join(checkpoint_dir, fname))
+
+# ============================================================================
+# LLaMA Factory Callback: integrates patch 2 (tokenizer copy) into the
+# training loop.
+# ============================================================================
+
+try:
+    from transformers import TrainerCallback
+    from transformers.trainer_utils import PREFIX_CHECKPOINT_DIR
+
+    class HYV4PatchCallback(TrainerCallback):
+        """
+        LLaMA Factory compatible callback that applies HYV4-specific patches.
+
+        Add to your YAML or pass to Trainer:
+            callbacks: [hy_v4_patches.HYV4PatchCallback]
+        """
+
+        def __init__(self, tokenizer_dir: Optional[str] = None):
+            """
+            Args:
+                tokenizer_dir: Path to the original tokenizer directory.
+                    If None, will try to use model_name_or_path from training args.
+            """
+            self._tokenizer_dir = tokenizer_dir
+
+        def on_save(self, args, state, control, **kwargs):
+            # --- Patch 2: Copy tokenizer files ---
+            if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0:
+                return control
+
+            checkpoint_dir = os.path.join(
+                args.output_dir,
+                f"{PREFIX_CHECKPOINT_DIR}-{state.global_step}",
+            )
+
+            # Determine tokenizer directory
+            tokenizer_dir = self._tokenizer_dir
+            if tokenizer_dir is None:
+                # Try common locations
+                tokenizer_dir = getattr(args, "tokenizer_name_or_path", None)
+                if tokenizer_dir is None:
+                    tokenizer_dir = getattr(args, "model_name_or_path", None)
+
+            if tokenizer_dir and os.path.isdir(tokenizer_dir):
+                _copy_tokenizer_to_checkpoint(tokenizer_dir, checkpoint_dir)
+                logger.info(
+                    "HYV4: Copied tokenizer files from %s to %s",
+                    tokenizer_dir, checkpoint_dir
+                )
+
+            return control
+
+except ImportError:
+    logger.warning(
+        "transformers not available; HYV4PatchCallback not defined."
+    )
+
+# ============================================================================
+# Patch 3: Memory-efficient shard-by-shard model loading for ZeRO-3
+#
+# The default transformers from_pretrained + ZeRO-3 path loads ALL shards
+# into a single state_dict in CPU memory before distributing. For a ~670GB
+# model with 8 processes per node, this causes CPU OOM.
+#
+# This patch replaces from_pretrained with a shard-by-shard loader that:
+#   1. Creates the model skeleton under deepspeed.zero.Init (meta tensors)
+#   2. Loads each safetensors shard one at a time (~7GB each)
+#   3. Applies key renames + expert fusion per shard
+#   4. Scatters into ZeRO-3 partitions immediately
+#   5. Frees the shard before loading the next one
+#
+# This reduces per-rank CPU memory from ~670GB to ~7GB.
+# ============================================================================
+
+def _apply_shard_loading_patch():
+    """Monkey-patch AutoModelForCausalLM.from_pretrained to use shard-by-shard
+    loading when DeepSpeed ZeRO-3 is active."""
+    import gc
+    import json as _json
+    import transformers
+
+    _orig_from_pretrained = transformers.AutoModelForCausalLM.from_pretrained
+
+    def _shard_loading_from_pretrained(pretrained_model_name_or_path, *args, **kwargs):
+        """Memory-efficient from_pretrained that loads shards one at a time."""
+
+        model_path = pretrained_model_name_or_path
+
+        # Helper: fallback to default from_pretrained with CPU-safe loading.
+        # When not using DeepSpeed ZeRO-3 (e.g. FSDP1 mode), we must avoid
+        # loading the full model onto GPU (which would OOM for large models).
+        # Force device_map to CPU; FSDP1 will handle GPU sharding after wrap.
+        def _fallback_load():
+            kwargs.setdefault("low_cpu_mem_usage", True)
+            # For FSDP mode: only local_rank 0 loads real weights to CPU;
+            # other ranks create model on meta device (zero CPU memory).
+            # FSDP's sync_module_states=True (default) will broadcast weights
+            # from rank 0 to all other ranks during wrap.
+            # This reduces per-node peak CPU memory from N_ranks * model_size
+            # to just 1 * model_size.
+            local_rank = int(os.environ.get("LOCAL_RANK", "0"))
+            if local_rank != 0:
+                logger.info(
+                    "[HYV4] FSDP mode: local_rank=%d != 0, "
+                    "creating model on meta device (zero CPU memory). "
+                    "Weights will be synced from rank 0 by FSDP.",
+                    local_rank,
+                )
+                import transformers
+                config = transformers.AutoConfig.from_pretrained(
+                    pretrained_model_name_or_path,
+                    trust_remote_code=kwargs.get("trust_remote_code", False),
+                )
+                torch_dtype = kwargs.get("torch_dtype", None)
+                if torch_dtype is None:
+                    torch_dtype = getattr(config, "torch_dtype", torch.bfloat16)
+                if not isinstance(torch_dtype, torch.dtype):
+                    torch_dtype = torch.bfloat16
+                with torch.device("meta"):
+                    model = transformers.AutoModelForCausalLM.from_config(
+                        config,
+                        torch_dtype=torch_dtype,
+                        trust_remote_code=kwargs.get("trust_remote_code", False),
+                    )
+                return model
+
+            # local_rank 0: load real weights to CPU.
+            # FSDP1 will handle GPU sharding after wrap.
+            logger.info(
+                "[HYV4] FSDP mode: local_rank=0, loading real weights to CPU."
+            )
+            # LLaMA Factory may pass device_map pointing to GPU, which
+            # would cause OOM for large models.
+            kwargs["device_map"] = {"": "cpu"}
+            return _orig_from_pretrained(
+                pretrained_model_name_or_path, *args, **kwargs
+            )
+
+        # Only apply shard loading if:
+        # 1. It's a local directory with safetensors
+        # 2. DeepSpeed ZeRO-3 is being used
+        if not (isinstance(model_path, str) and os.path.isdir(model_path)):
+            return _fallback_load()
+
+        index_file = os.path.join(model_path, "model.safetensors.index.json")
+        single_file = os.path.join(model_path, "model.safetensors")
+        if not (os.path.isfile(index_file) or os.path.isfile(single_file)):
+            return _fallback_load()
+
+        # Try to determine the DeepSpeed config
+        ds_config = None
+
+        # Check if there's a deepspeed config in the HfTrainerDeepSpeedConfig
+        try:
+            from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled
+            if not is_deepspeed_zero3_enabled():
+                logger.info(
+                    "[HYV4 Patch 3] ZeRO-3 not enabled, using CPU fallback loader."
+                )
+                return _fallback_load()
+        except (ImportError, Exception):
+            # If we can't determine, try to proceed anyway
+            pass
+
+        # Get the deepspeed config from HF's global state
+        try:
+            from transformers.integrations.deepspeed import deepspeed_config as _get_ds_config
+            ds_config = _get_ds_config()
+        except (ImportError, Exception):
+            ds_config = None
+
+        if ds_config is None:
+            # Fallback: try weak ref approach
+            try:
+                from transformers.integrations import deepspeed as _hf_ds
+                if hasattr(_hf_ds, '_hf_deepspeed_config_weak_ref'):
+                    _weak_ref = _hf_ds._hf_deepspeed_config_weak_ref
+                    if _weak_ref is not None:
+                        ds_obj = _weak_ref()
+                        if ds_obj is not None:
+                            ds_config = ds_obj.config
+            except (ImportError, AttributeError, Exception):
+                pass
+
+        if ds_config is None:
+            # Last resort: look for the config file path in environment
+            ds_config_path = os.environ.get("DEEPSPEED_CONFIG_FILE", None)
+            if ds_config_path is None:
+                ds_config_path = os.environ.get("DEEPSPEED_CONFIG", None)
+            if ds_config_path and os.path.isfile(ds_config_path):
+                with open(ds_config_path, "r") as f:
+                    ds_config = _json.load(f)
+
+        if ds_config is None:
+            logger.warning(
+                "[HYV4 Patch 3] Cannot determine DeepSpeed config, "
+                "falling back to CPU loader (FSDP mode)."
+            )
+            return _fallback_load()
+
+        # Ensure ds_config is a dict
+        if hasattr(ds_config, 'config'):
+            ds_config = ds_config.config
+        if not isinstance(ds_config, dict):
+            logger.warning(
+                "[HYV4 Patch 3] ds_config is not a dict (%s), falling back.",
+                type(ds_config)
+            )
+            return _fallback_load()
+
+        # Check if it's actually ZeRO stage 3
+        zero_stage = ds_config.get("zero_optimization", {}).get("stage", 0)
+        if zero_stage != 3:
+            logger.info(
+                "[HYV4 Patch 3] Not ZeRO-3 (stage=%d), using CPU fallback loader.",
+                zero_stage
+            )
+            return _fallback_load()
+
+        logger.info(
+            "[HYV4 Patch 3] Using shard-by-shard loading for model at: %s",
+            model_path
+        )
+
+        import deepspeed
+
+        try:
+            from safetensors import safe_open
+            from transformers.integrations.deepspeed import (
+                _load_state_dict_into_zero3_model as _load_zero3,
+            )
+        except ImportError as e:
+            logger.warning(
+                "[HYV4 Patch 3] Required imports not available (%s), "
+                "falling back to default from_pretrained.", e
+            )
+            return _orig_from_pretrained(pretrained_model_name_or_path, *args, **kwargs)
+
+        # Replace "auto" values that deepspeed.zero.Init cannot resolve
+        ds_config_copy = _json.loads(_json.dumps(ds_config))
+        _auto_defaults = {
+            "train_batch_size": 32,
+            "train_micro_batch_size_per_gpu": 1,
+            "gradient_accumulation_steps": 1,
+            "gradient_clipping": 1.0,
+        }
+        for k, v in _auto_defaults.items():
+            if k in ds_config_copy and ds_config_copy[k] == "auto":
+                ds_config_copy[k] = v
+
+        # Determine dtype
+        torch_dtype = kwargs.pop("torch_dtype", torch.bfloat16)
+        if torch_dtype is None or torch_dtype == "auto":
+            torch_dtype = torch.bfloat16
+        trust_remote_code = kwargs.pop("trust_remote_code", True)
+        attn_implementation = kwargs.pop("attn_implementation", None)
+        # Pop config if already provided by caller (e.g. LLaMA Factory)
+        config = kwargs.pop("config", None)
+
+        # Step 1: Create model skeleton under ZeRO-3 Init (meta tensors)
+        if config is None:
+            config = transformers.AutoConfig.from_pretrained(
+                model_path, trust_remote_code=trust_remote_code
+            )
+        with deepspeed.zero.Init(
+            dtype=torch_dtype, config_dict_or_path=ds_config_copy
+        ):
+            model = transformers.AutoModelForCausalLM.from_config(
+                config,
+                trust_remote_code=trust_remote_code,
+                torch_dtype=torch_dtype,
+                attn_implementation=attn_implementation,
+            )
+        logger.info("[HYV4 Patch 3] Model skeleton created under ZeRO-3 Init.")
+
+        # Step 2: Determine shard files
+        if os.path.isfile(index_file):
+            with open(index_file, "r") as f:
+                index_data = _json.load(f)
+            shard_files = list(dict.fromkeys(index_data["weight_map"].values()))
+        else:
+            shard_files = ["model.safetensors"]
+
+        # Step 3: Load each shard and scatter into ZeRO-3 model
+        total_shards = len(shard_files)
+        all_loaded_keys = set()
+        pending_experts = {}  # prefix -> {expert_idx -> {proj_name -> tensor}}
+
+        for shard_idx, shard_name in enumerate(shard_files, 1):
+            shard_path = os.path.join(model_path, shard_name)
+            logger.info(
+                "[HYV4 Patch 3] Loading shard %d/%d: %s",
+                shard_idx, total_shards, shard_name
+            )
+
+            # Load shard into CPU memory
+            shard_sd = {}
+            with safe_open(shard_path, framework="pt", device="cpu") as f:
+                for key in f.keys():
+                    shard_sd[key] = f.get_tensor(key)
+
+            # Separate expert keys from non-expert keys, apply renames
+            renamed_sd = {}
+            expert_keys_in_shard = {}
+
+            for k, v in shard_sd.items():
+                m = _EXPERT_KEY_RE.match(k)
+                if m:
+                    prefix = m.group(1)
+                    expert_idx = int(m.group(2))
+                    proj_name = m.group(3)
+                    if prefix not in expert_keys_in_shard:
+                        expert_keys_in_shard[prefix] = {}
+                    if expert_idx not in expert_keys_in_shard[prefix]:
+                        expert_keys_in_shard[prefix][expert_idx] = {}
+                    expert_keys_in_shard[prefix][expert_idx][proj_name] = v
+                else:
+                    new_k = k
+                    for old_sub, new_sub in _CKPT_KEY_RENAMES:
+                        if old_sub in new_k:
+                            new_k = new_k.replace(old_sub, new_sub)
+                    renamed_sd[new_k] = v
+            del shard_sd
+
+            # Merge expert keys into pending_experts
+            for prefix, experts in expert_keys_in_shard.items():
+                if prefix not in pending_experts:
+                    pending_experts[prefix] = {}
+                for idx, projs in experts.items():
+                    if idx not in pending_experts[prefix]:
+                        pending_experts[prefix][idx] = {}
+                    pending_experts[prefix][idx].update(projs)
+            del expert_keys_in_shard
+
+            # Check for completed expert groups
+            completed_prefixes = []
+            for prefix, experts in pending_experts.items():
+                if not experts:
+                    continue
+                max_idx = max(experts.keys())
+                num_experts_found = len(experts)
+                all_complete = all(
+                    len(projs) == 3 for projs in experts.values()
+                )
+                if all_complete and num_experts_found == (max_idx + 1):
+                    completed_prefixes.append(prefix)
+
+            # Fuse completed expert groups
+            for prefix in completed_prefixes:
+                experts = pending_experts.pop(prefix)
+                num_experts_layer = max(experts.keys()) + 1
+                gate_up_list = []
+                down_list = []
+                for i in range(num_experts_layer):
+                    exp = experts[i]
+                    gate_up = torch.cat([exp["gate_proj"], exp["up_proj"]], dim=0)
+                    gate_up_list.append(gate_up)
+                    down_list.append(exp["down_proj"])
+                fused_gate_up = torch.stack(gate_up_list, dim=0)
+                fused_down = torch.stack(down_list, dim=0)
+                del gate_up_list, down_list, experts
+                renamed_sd[f"{prefix}gate_up_proj"] = fused_gate_up
+                renamed_sd[f"{prefix}down_proj"] = fused_down
+                logger.info(
+                    "[HYV4 Patch 3]   Fused %d experts for %s",
+                    num_experts_layer, prefix
+                )
+
+            # Scatter this shard's weights into ZeRO-3 model
+            if renamed_sd:
+                _load_zero3(model, renamed_sd)
+                # Also load buffers
+                for name, buf in model.named_buffers():
+                    if name in renamed_sd:
+                        src_tensor = renamed_sd[name]
+                        if isinstance(src_tensor, torch.Tensor):
+                            buf.data.copy_(src_tensor.to(buf.dtype))
+                all_loaded_keys.update(renamed_sd.keys())
+            del renamed_sd
+            gc.collect()
+
+        # Flush remaining pending experts
+        if pending_experts:
+            logger.info(
+                "[HYV4 Patch 3] Flushing %d remaining expert group(s)...",
+                len(pending_experts)
+            )
+            flush_sd = {}
+            for prefix, experts in pending_experts.items():
+                num_experts_layer = max(experts.keys()) + 1
+                gate_up_list = []
+                down_list = []
+                for i in range(num_experts_layer):
+                    if i not in experts:
+                        logger.warning(
+                            "[HYV4 Patch 3] Missing expert %d in %s", i, prefix
+                        )
+                        continue
+                    exp = experts[i]
+                    gate_up = torch.cat([exp["gate_proj"], exp["up_proj"]], dim=0)
+                    gate_up_list.append(gate_up)
+                    down_list.append(exp["down_proj"])
+                if gate_up_list:
+                    fused_gate_up = torch.stack(gate_up_list, dim=0)
+                    fused_down = torch.stack(down_list, dim=0)
+                    flush_sd[f"{prefix}gate_up_proj"] = fused_gate_up
+                    flush_sd[f"{prefix}down_proj"] = fused_down
+                    logger.info(
+                        "[HYV4 Patch 3]   Fused %d experts for %s",
+                        len(gate_up_list), prefix
+                    )
+                del gate_up_list, down_list
+            del pending_experts
+
+            if flush_sd:
+                _load_zero3(model, flush_sd)
+                for name, buf in model.named_buffers():
+                    if name in flush_sd:
+                        src_tensor = flush_sd[name]
+                        if isinstance(src_tensor, torch.Tensor):
+                            buf.data.copy_(src_tensor.to(buf.dtype))
+                all_loaded_keys.update(flush_sd.keys())
+            del flush_sd
+            gc.collect()
+
+        # Report missing/unexpected keys
+        model_keys = set(n for n, _ in model.named_parameters())
+        model_keys.update(n for n, _ in model.named_buffers())
+        missing = model_keys - all_loaded_keys
+        unexpected = all_loaded_keys - model_keys
+        if missing:
+            real_missing = {k for k in missing if "lm_head" not in k}
+            if real_missing:
+                logger.warning(
+                    "[HYV4 Patch 3] %d keys not found in checkpoint (first 10): %s",
+                    len(real_missing), list(real_missing)[:10]
+                )
+        if unexpected:
+            logger.warning(
+                "[HYV4 Patch 3] %d unexpected keys (first 10): %s",
+                len(unexpected), list(unexpected)[:10]
+            )
+        logger.info(
+            "[HYV4 Patch 3] Shard-by-shard loading complete. "
+            "Loaded %d keys from %d shards.",
+            len(all_loaded_keys), total_shards
+        )
+
+        return model
+
+    # Apply the monkey-patch
+    transformers.AutoModelForCausalLM.from_pretrained = staticmethod(_shard_loading_from_pretrained)
+    logger.info(
+        "HYV4 patch applied: shard-by-shard model loading for ZeRO-3 "
+        "(reduces CPU memory from ~670GB to ~7GB per rank)."
+    )
+
+
+# ============================================================================
+# Auto-apply patches on import
+# ============================================================================
+
+# Patch 1: ZeRO-3 key rename + expert fuse + buffer loading
+_apply_buffer_loading_patch()
+
+# Patch 3: Memory-efficient shard-by-shard loading
+_apply_shard_loading_patch()
+
+# Patch 4: Unify model dtype before FSDP wrap
+# FSDP1 requires all parameters in the same FSDP unit to have uniform dtype.
+# After LoRA injection, adapter params are float32 while base model is bfloat16.
+# This patch casts all parameters to bf16 before FSDP wraps the model.
+def _apply_fsdp_dtype_patch():
+    """Monkey-patch Trainer to cast model to bf16 before FSDP wrap."""
+    try:
+        from transformers import Trainer
+
+        _orig_prepare_for_training = Trainer._prepare_for_training
+
+        def _patched_prepare_for_training(self, *args, **kwargs):
+            # Cast all model parameters to bf16 before FSDP wrap and disable
+            # mixed precision to prevent Accelerate from upcasting bf16→fp32.
+            if getattr(self.args, 'fsdp', False):
+                model = self.model
+                dtype_counts = {}
+                for p in model.parameters():
+                    dt = str(p.dtype)
+                    dtype_counts[dt] = dtype_counts.get(dt, 0) + 1
+
+                if len(dtype_counts) > 1:
+                    logger.info(
+                        "[HYV4 Patch 4] Mixed dtypes detected before FSDP wrap: %s. "
+                        "Casting all parameters to bfloat16.",
+                        dtype_counts
+                    )
+                    for p in model.parameters():
+                        if p.dtype != torch.bfloat16:
+                            p.data = p.data.to(torch.bfloat16)
+                else:
+                    logger.info(
+                        "[HYV4 Patch 4] All params already uniform dtype: %s",
+                        dtype_counts
+                    )
+
+                # Disable FSDP mixed precision to prevent Accelerate from
+                # upcasting bf16 params to fp32 during wrap. Since all params
+                # are already bf16 (cast above), we don't need FSDP's mixed
+                # precision policy. This saves ~2x GPU memory per shard.
+                try:
+                    if hasattr(self, 'accelerator'):
+                        # Level 1: Set accelerator state _mixed_precision to "no"
+                        # Note: mixed_precision is a property, must set _mixed_precision
+                        if hasattr(self.accelerator, 'state'):
+                            old_mp = getattr(self.accelerator.state, '_mixed_precision', None)
+                            self.accelerator.state._mixed_precision = "no"
+                            logger.info(
+                                "[HYV4 Patch 4] Set accelerator.state._mixed_precision='no' "
+                                "(was: %s) to prevent bf16→fp32 upcast.", old_mp
+                            )
+                        # Level 2: Clear the FSDP plugin's mixed_precision_policy
+                        fsdp_plugin = getattr(self.accelerator.state, 'fsdp_plugin', None)
+                        if fsdp_plugin is not None:
+                            if hasattr(fsdp_plugin, 'mixed_precision_policy'):
+                                fsdp_plugin.mixed_precision_policy = None
+                            if hasattr(fsdp_plugin, 'kwargs'):
+                                fsdp_plugin.kwargs.pop('mixed_precision', None)
+                            logger.info(
+                                "[HYV4 Patch 4] Cleared fsdp_plugin mixed_precision_policy."
+                            )
+                    else:
+                        logger.warning(
+                            "[HYV4 Patch 4] self.accelerator not found, cannot disable mixed precision."
+                        )
+                    # Level 3: Environment variable (for any lazy initialization)
+                    os.environ["ACCELERATE_MIXED_PRECISION"] = "no"
+                except Exception as e:
+                    logger.warning(
+                        "[HYV4 Patch 4] Failed to disable mixed precision: %s", e
+                    )
+
+            return _orig_prepare_for_training(self, *args, **kwargs)
+
+        Trainer._prepare_for_training = _patched_prepare_for_training
+        logger.info(
+            "HYV4 Patch 4 applied: unify model dtype to bf16 before FSDP wrap."
+        )
+    except (ImportError, AttributeError) as e:
+        logger.warning("[HYV4 Patch 4] Could not apply dtype patch: %s", e)
+
+_apply_fsdp_dtype_patch()
+
+# Patch 5: Fix CustomSeq2SeqTrainer.create_optimizer signature
+# New transformers (5.15+) calls self.create_optimizer(model) with a model arg,
+# but LLaMA Factory's CustomSeq2SeqTrainer.create_optimizer(self) doesn't accept it.
+def _apply_create_optimizer_patch():
+    """Monkey-patch CustomSeq2SeqTrainer.create_optimizer to accept optional model arg."""
+    try:
+        from llamafactory.train.sft.trainer import CustomSeq2SeqTrainer
+
+        _orig_create_optimizer = CustomSeq2SeqTrainer.create_optimizer
+
+        def _patched_create_optimizer(self, model=None):
+            return _orig_create_optimizer(self)
+
+        CustomSeq2SeqTrainer.create_optimizer = _patched_create_optimizer
+        logger.info(
+            "HYV4 Patch 5 applied: CustomSeq2SeqTrainer.create_optimizer "
+            "now accepts optional model argument for transformers >= 5.15."
+        )
+    except (ImportError, AttributeError) as e:
+        logger.warning("[HYV4 Patch 5] Could not apply create_optimizer patch: %s", e)
+
+_apply_create_optimizer_patch()
+
+# Patch 2 (tokenizer copy) is applied via HYV4PatchCallback during training.
+# Users should add HYV4PatchCallback to their Trainer callbacks.
+
+logger.info(
+    "HYV4 patches module loaded. Remember to add HYV4PatchCallback to "
+    "your Trainer callbacks for tokenizer file copy on save."
+)

finetune/llama_factory_support/hy_v4_template.py

@@ -0,0 +1,50 @@
+"""
+HYV4 chat template registration for LLaMA Factory.
+
+Usage:
+    1. Copy this file's register_template block into LLaMA Factory's
+       src/llamafactory/data/template.py  (for upstream MR).
+    2. Or import this module before training to register at runtime:
+       import hy_v4_template
+"""
+
+from llamafactory.data.template import ReasoningTemplate, register_template
+from llamafactory.data.formatter import EmptyFormatter, StringFormatter
+
+
+# ---------------------------------------------------------------------------
+# HYV4 (MoE, pure text) chat template
+#
+# Token format (from chat_template.jinja & tokenizer_config.json):
+#   Each turn:  <｜hy_start:opensource｜>{role}<｜hy_middle:opensource｜>{content}<｜hy_end:opensource｜>
+#   BOS:        <｜hy_start:opensource｜>  (token ID 120000)
+#   Middle:     <｜hy_middle:opensource｜> (token ID 120001)
+#   EOS:        <｜hy_end:opensource｜>    (token ID 120025)
+#
+# Loss mask: only compute loss on assistant content (including eos).
+#
+# Reasoning / Slow-thinking support:
+#   This template uses ReasoningTemplate with thought_words so that LLaMA
+#   Factory can correctly mask think-tag tokens during loss computation.
+#   - thought_words: ("<think:opensource>", "</think:opensource>")
+#   - enable_thinking: set globally via data_args.enable_thinking (default True)
+#
+#   IMPORTANT: To train slow-thinking (chain-of-thought) behaviour, your
+#   training data must include <think:opensource>...</think:opensource> tags
+#   inside the assistant content.  If your data does NOT contain think tags,
+#   the model will only learn fast-thinking (direct answer) mode.
+#   The `reasoning_effort` field from the API is NOT used by LLaMA Factory;
+#   slow-vs-fast is determined solely by the presence of think tags in data.
+# ---------------------------------------------------------------------------
+
+register_template(
+    name="hy_v4",
+    template_class=ReasoningTemplate,
+    format_user=StringFormatter(slots=["<｜hy_start:opensource｜>user<｜hy_middle:opensource｜>{{content}}<｜hy_end:opensource｜>"]),
+    format_assistant=StringFormatter(slots=["<｜hy_start:opensource｜>assistant<｜hy_middle:opensource｜>{{content}}<｜hy_end:opensource｜>"]),
+    format_system=StringFormatter(slots=["<｜hy_start:opensource｜>system<｜hy_middle:opensource｜>{{content}}<｜hy_end:opensource｜>"]),
+    format_prefix=EmptyFormatter(slots=[]),
+    thought_words=("<think:opensource>", "</think:opensource>"),
+    stop_words=["<｜hy_end:opensource｜>"],
+    efficient_eos=False,
+)

finetune/llama_factory_support/train_hy_v4.py

@@ -0,0 +1,99 @@
+"""
+LLaMA Factory training entry-point wrapper.
+
+This script:
+  1. Registers the chat template
+  2. Applies all monkey-patches (checkpoint key rename, dtype fix, etc.)
+  3. Injects HYV4PatchCallback into the training loop
+  4. Calls run_exp() to start LLaMA Factory training
+
+"""
+
+import sys
+import os
+
+# Add current directory to path so patches can be imported
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+# Step 1: Register HYV4 template (must be before training starts)
+import hy_v4_template  # noqa: F401
+
+# Step 2: Apply checkpoint key rename patch (must be before model loading)
+import hy_v4_patches  # noqa: F401
+
+# Step 3: Inject HYV4PatchCallback into LLaMA Factory's training flow
+from llamafactory.train.sft.workflow import run_sft as _orig_run_sft
+
+
+def _patched_run_sft(
+    model_args, data_args, training_args,
+    finetuning_args, generating_args, callbacks=None
+):
+    """Wrap run_sft to inject HYV4PatchCallback."""
+    if callbacks is None:
+        callbacks = []
+
+    # Determine tokenizer directory for the save callback
+    tokenizer_dir = getattr(model_args, "model_name_or_path", None)
+    callbacks.append(
+        hy_v4_patches.HYV4PatchCallback(tokenizer_dir=tokenizer_dir)
+    )
+
+    return _orig_run_sft(
+        model_args, data_args, training_args,
+        finetuning_args, generating_args,
+        callbacks=callbacks
+    )
+
+
+# Monkey-patch the SFT workflow
+import llamafactory.train.sft.workflow as _sft_wf
+_sft_wf.run_sft = _patched_run_sft
+
+
+def _apply_skip_grad_norm_patch():
+    """Skip grad norm computation for DeepSpeed ZeRO-3 + CPU offload.
+
+    Under ZeRO-3 + CPU offload, DeepSpeed's complete_grad_norm_calculation
+    all-gathers every gradient on CPU and does an ALLREDUCE to compute the
+    global L2 norm. For a 770B model this is extremely slow and can cause
+    NCCL timeout/deadlock at optimizer step.
+
+    When max_grad_norm=0 (no clipping), we fully skip the norm computation
+    by patching _get_norm_groups to return 0.0 immediately.
+    """
+    import torch
+    try:
+        from deepspeed.runtime.zero.stage3 import DeepSpeedZeroOptimizer_Stage3
+
+        def _skip_get_norm_groups(self):
+            return [torch.tensor(0.0)]
+
+        DeepSpeedZeroOptimizer_Stage3._get_norm_groups = _skip_get_norm_groups
+        print("[HYV4 Patch] Patched DeepSpeedZeroOptimizer_Stage3._get_norm_groups "
+              "to skip grad norm computation (max_grad_norm=0).", flush=True)
+    except ImportError:
+        # DeepSpeed not available, skip
+        pass
+
+
+def main():
+    """Entry point: called by torchrun in each worker process.
+
+    Since train_lf.sh launches us via torchrun directly, all patches
+    (template registration, checkpoint key rename, SFT callback injection)
+    are already applied in this process.  We just call run_exp() to start
+    training — no need to go through the CLI launcher.
+    """
+    # Apply grad norm skip patch for DeepSpeed ZeRO-3 full SFT.
+    # This must be done before Trainer creates the DeepSpeed engine.
+    # The patch is safe even when not using DeepSpeed (it's a no-op if
+    # DeepSpeed is not imported or ZeRO-3 is not used).
+    _apply_skip_grad_norm_patch()
+
+    from llamafactory.train.tuner import run_exp
+    run_exp()
+
+
+if __name__ == "__main__":
+    main()

finetune/llama_factory_support/train_lf.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# -------------------- Network Configuration --------------------
+NET_TYPE="high"
+export NCCL_DEBUG=WARN
+export NCCL_P2P_LEVEL=NVL
+export NCCL_IB_TIMEOUT=24
+export NCCL_NVLS_ENABLE=0
+export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
+if [[ "${NET_TYPE}" = "low" ]]; then
+    export NCCL_SOCKET_IFNAME=eth1
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_HCA=mlx5_2:1
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+else
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_IB_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+    export NCCL_SOCKET_IFNAME=bond1
+    export UCX_NET_DEVICES=bond1
+    export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
+    export NCCL_COLLNET_ENABLE=0
+    export SHARP_COLL_ENABLE_SAT=0
+    export NCCL_NET_GDR_LEVEL=2
+    export NCCL_IB_QPS_PER_CONNECTION=4
+    export NCCL_IB_TC=160
+    export NCCL_PXN_DISABLE=1
+fi
+
+export DISABLE_VERSION_CHECK=1
+
+# -------------------- Node Configuration --------------------
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
+export IP_LIST=${IP_LIST:-"127.0.0.1"}
+
+MASTER_PORT=${MASTER_PORT:-29500}
+
+IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
+NODES=${#IP_ARRAY[@]}
+MASTER_ADDR=${IP_ARRAY[0]}
+
+# -------------------- Paths --------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+YAML_FILE="${YAML_FILE:-hy_v4_full_sft.yaml}"
+# If YAML_FILE is not an absolute path, resolve it relative to SCRIPT_DIR
+if [[ "${YAML_FILE}" != /* ]]; then
+    YAML_FILE="${SCRIPT_DIR}/${YAML_FILE}"
+fi
+ENTRY_SCRIPT="${SCRIPT_DIR}/train_hy_v4.py"
+
+# -------------------- Distributed Environment --------------------
+export MASTER_ADDR="${MASTER_ADDR}"
+export MASTER_PORT="${MASTER_PORT}"
+export NNODES="${NODES}"
+
+# Prevent Accelerate/FSDP from upcasting bf16 parameters back to fp32
+export ACCELERATE_MIXED_PRECISION=no
+
+if [ ${NODES} -gt 1 ]; then
+    # Determine local node rank by matching local IP against IP_LIST
+    LOCAL_IP=$(hostname -i | awk '{print $1}')
+    NODE_RANK=0
+    for i in "${!IP_ARRAY[@]}"; do
+        if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
+            NODE_RANK=$i
+            break
+        fi
+    done
+    export RANK="${NODE_RANK}"
+else
+    export RANK=0
+fi
+
+echo "============================================"
+echo "  HYV4 LLaMA Factory Training"
+echo "  Nodes: ${NNODES}, Rank: ${RANK}"
+echo "  Master: ${MASTER_ADDR}:${MASTER_PORT}"
+echo "  GPUs per node: ${HOST_GPU_NUM}"
+echo "  Total GPUs: $((NODES * HOST_GPU_NUM))"
+echo "============================================"
+
+# -------------------- Launch --------------------
+# We launch torchrun directly (instead of FORCE_TORCHRUN) so that each
+# worker process runs train_hy_v4.py with all HYV4 patches applied.
+torchrun \
+    --nnodes "${NNODES}" \
+    --node_rank "${RANK}" \
+    --nproc_per_node "${HOST_GPU_NUM}" \
+    --master_addr "${MASTER_ADDR}" \
+    --master_port "${MASTER_PORT}" \
+    "${ENTRY_SCRIPT}" "${YAML_FILE}"

finetune/ms_swift_support/fsdp_hy_v4.json

@@ -0,0 +1,20 @@
+{
+"_description": "FSDP configuration",
+    "_note": "Uses FSDP with full_shard auto_wrap.",
+
+    "fsdp": "full_shard auto_wrap",
+    "fsdp_config": {
+        "version": 1,
+        "reshard_after_forward": "full_shard",
+        "auto_wrap_policy": "TRANSFORMER_BASED_WRAP",
+        "transformer_layer_cls_to_wrap": ["HYV4DecoderLayer"],
+        "cpu_ram_efficient_loading": true,
+        "state_dict_type": "FULL_STATE_DICT",
+        "activation_checkpointing": true,
+        "use_orig_params": true,
+        "sync_module_states": true,
+        "forward_prefetch": false,
+        "backward_prefetch": "NO_PREFETCH",
+        "limit_all_gathers": true
+    }
+}

finetune/ms_swift_support/hy_v4_full_sft.yaml

@@ -0,0 +1,63 @@
+
+# ---- Model Settings ----
+model: /path/to/hy_ckpt
+model_type: hy_v4
+template: hy_v4
+torch_dtype: bfloat16
+attn_impl: sdpa
+trust_remote_code: true
+
+# ---- Training Method ----
+train_type: full
+
+# ---- Dataset Settings ----
+dataset:
+  - ../data/example_data.jsonl
+max_length: 256
+lazy_tokenize: true
+dataset_num_proc: 4
+
+# ---- Output Settings ----
+output_dir: saves/hy_v4/full/sft_ds
+save_steps: 500
+save_strategy: steps
+save_total_limit: 3
+save_only_model: false
+logging_steps: 1
+report_to: none
+
+# ---- Training Hyperparameters ----
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 1.0e-5
+num_train_epochs: 3.0
+warmup_steps: 0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1
+bf16: true
+
+# ---- DeepSpeed ZeRO-3 + CPU Offload ----
+# Use custom ds_config for fine-grained control over offload settings
+deepspeed: ../deepspeed_support/ds_zero3_offload.json
+
+# ---- Optimization ----
+# adamw_torch is required: adafactor's tuple eps is incompatible with
+# DeepSpeed's CPU Adam kernel (causes TypeError: adam_update())
+optim: adamw_torch
+gradient_checkpointing: true
+# use_reentrant=true is required for DeepSpeed ZeRO-3 to avoid
+# CheckpointError due to parameter partition state changes during recompute
+gradient_checkpointing_kwargs:
+  use_reentrant: true
+# max_grad_norm=0 skips grad norm computation (patched in hy_v4_swift_patches.py)
+# to avoid NCCL ALLREDUCE timeout on 770B model
+max_grad_norm: 0.0
+
+# ---- Distributed Training ----
+ddp_timeout: 180000000
+
+# ---- Misc ----
+seed: 42
+ignore_data_skip: true
+resume_from_checkpoint: null

finetune/ms_swift_support/hy_v4_lora_sft.yaml

@@ -0,0 +1,65 @@
+
+# ---- Model Settings ----
+model: /path/to/hy_ckpt
+model_type: hy_v4
+template: hy_v4
+torch_dtype: bfloat16
+attn_impl: sdpa
+trust_remote_code: true
+
+# ---- Training Method: LoRA ----
+train_type: lora
+lora_rank: 64
+lora_alpha: 128
+lora_dropout: 0.05
+# HYV4 uses MLA (Multi-head Latent Attention) with these projection names
+target_modules:
+  - q_a_proj
+  - q_b_proj
+  - kv_a_proj_with_mqa
+  - kv_b_proj
+  - o_proj
+
+# ---- Dataset Settings ----
+dataset:
+  - ../data/example_data.jsonl
+max_length: 1024
+lazy_tokenize: true
+dataset_num_proc: 4
+
+# ---- Output Settings ----
+output_dir: saves/hy_v4/lora/sft_fsdp
+save_steps: 500
+save_strategy: steps
+save_total_limit: 3
+save_only_model: false
+logging_steps: 1
+report_to: none
+
+# ---- Training Hyperparameters ----
+per_device_train_batch_size: 1
+gradient_accumulation_steps: 1
+learning_rate: 2.0e-4
+num_train_epochs: 3.0
+warmup_steps: 0
+lr_scheduler_type: cosine_with_min_lr
+lr_scheduler_kwargs:
+  min_lr_rate: 0.1
+bf16: true
+
+# ---- FSDP Configuration ----
+# ms-swift uses FSDP (PyTorch native). Pass the JSON config file path to --fsdp.
+# FSDP uses activation_checkpointing (configured in JSON) instead of gradient_checkpointing.
+fsdp: fsdp_hy_v4.json
+
+# ---- Optimization ----
+optim: adamw_torch
+max_grad_norm: 1.0
+
+# ---- Distributed Training ----
+ddp_timeout: 180000000
+
+# ---- Misc ----
+seed: 42
+ignore_data_skip: true
+resume_from_checkpoint: null

finetune/ms_swift_support/hy_v4_swift_patches.py

@@ -0,0 +1,431 @@
+"""
+Patches for ms-swift training.
+
+Patches auto-applied on import:
+    1. Model & Template registration: Register the custom model_type
+       in ms-swift.
+    2. Grad norm skip patch: Skip grad norm computation for ZeRO-3 + CPU offload.
+    3. Memory-efficient model loading: Shard-by-shard loading that
+       processes one safetensors shard for memory optimization.
+    4. Fix logging_dir compatibility between ms-swift and transformers 5.x.
+    5. Align FSDP1 dtype / mixed precision behavior with LLaMA Factory.
+    6. Disable _compute_acc to avoid errors during training.
+
+Optional (call manually after LoRA is applied):
+    - apply_lora_z3_leaf_patch(model): Mark PEFT LoRA wrapper modules as
+      ZeRO-3 leaf modules to fix parameter fetch/release scheduling issues.
+
+Usage:
+    swift sft --custom_register_path hy_v4_swift_patches.py --model /path/to/ckpt ...
+"""
+
+import os
+import gc
+import json as _json
+import logging
+from typing import Any, Dict
+
+import torch
+
+logger = logging.getLogger(__name__)
+
+# ============================================================================
+# Patch 1: Model & Template Registration
+#
+# ms-swift natively supports hy_v3, but HYV4 has a different model class
+# (HYV4ForCausalLM) and additional iHC modules. We register HYV4 as a
+# custom model_type that reuses the hy_v3 template.
+# ============================================================================
+
+from swift.model import register_model, ModelMeta, ModelGroup, Model
+from swift.template import register_template, TemplateMeta
+
+# Register hy_v4 template
+# Token format: <｜hy_start:opensource｜>{role}<｜hy_middle:opensource｜>{content}<｜hy_end:opensource｜>
+register_template(
+    TemplateMeta(
+        template_type='hy_v4',
+        prefix=[],
+        system_prefix=['<｜hy_start:opensource｜>system<｜hy_middle:opensource｜>{{SYSTEM}}<｜hy_end:opensource｜>'],
+        prompt=['<｜hy_start:opensource｜>user<｜hy_middle:opensource｜>{{QUERY}}<｜hy_end:opensource｜><｜hy_start:opensource｜>assistant<｜hy_middle:opensource｜>'],
+        chat_sep=['<｜hy_end:opensource｜>'],
+        suffix=['<｜hy_end:opensource｜>'],
+    ),
+    exist_ok=True,
+)
+
+# Register hy_v4 model
+register_model(
+    ModelMeta(
+        model_type='hy_v4',
+        model_groups=[
+            ModelGroup([
+                Model('Tencent-Hunyuan/Hy4',
+                      'Tencent-Hunyuan/Hy4'),
+            ]),
+        ],
+        template='hy_v4',
+        architectures=['HYV4ForCausalLM'],
+        is_multimodal=False,
+    ),
+    exist_ok=True,
+)
+
+logger.info(
+    "[HYV4 Patch 1] Model type 'hy_v4' and template registered in ms-swift."
+)
+
+# ============================================================================
+# Patch 2: Skip grad norm computation for DeepSpeed ZeRO-3
+#
+# Under ZeRO-3 + CPU offload, DeepSpeed's complete_grad_norm_calculation
+# all-gathers every gradient on CPU and does an ALLREDUCE to compute the
+# global L2 norm. For a 770B model this causes NCCL timeout/deadlock.
+# When max_grad_norm=0 (no clipping), we skip the norm computation entirely.
+# ============================================================================
+
+def _apply_skip_grad_norm_patch():
+    """Patch DeepSpeed ZeRO-3 optimizer to skip grad norm computation."""
+    try:
+        from deepspeed.runtime.zero.stage3 import DeepSpeedZeroOptimizer_Stage3
+
+        def _skip_get_norm_groups(self):
+            return [torch.tensor(0.0)]
+
+        DeepSpeedZeroOptimizer_Stage3._get_norm_groups = _skip_get_norm_groups
+        logger.info(
+            "[HYV4 Patch 2] Patched DeepSpeedZeroOptimizer_Stage3._get_norm_groups "
+            "to skip grad norm computation."
+        )
+    except ImportError:
+        logger.info("[HYV4 Patch 2] DeepSpeed not available, skipping grad norm patch.")
+
+
+# ============================================================================
+# Patch 3: Memory-efficient model loading for ZeRO-3 and FSDP1
+#
+# ZeRO-3 path:
+#   Ensure transformers sees the DeepSpeed config early enough to activate its
+#   native efficient loading path.
+#
+# FSDP1 path:
+#   Mirror the previously successful LLaMA Factory behavior:
+#     1. local_rank 0 loads real weights to CPU only
+#     2. all other local ranks create the model on meta device
+#     3. low_cpu_mem_usage stays enabled
+#
+# This avoids loading the full model onto every GPU during `from_pretrained`,
+# which is exactly the failure mode we observed with ms-swift + FSDP.
+# ============================================================================
+
+def _apply_shard_loading_patch():
+    """Ensure efficient model loading is used for both ZeRO-3 and FSDP1."""
+    import sys
+    from transformers import AutoConfig, PreTrainedModel
+
+    _real_orig_from_pretrained = PreTrainedModel.from_pretrained.__func__
+
+    def _disable_router_logits_if_needed(model):
+        if hasattr(model, 'config') and getattr(model.config, 'output_router_logits', False):
+            model.config.output_router_logits = False
+            print("[HYV4 Patch 3] Disabled output_router_logits.", flush=True)
+        return model
+
+    def _is_fsdp_requested():
+        accel_fsdp = str(os.environ.get("ACCELERATE_USE_FSDP", "")).lower()
+        if accel_fsdp in {"1", "true", "yes"}:
+            return True
+        return "--fsdp" in sys.argv
+
+    def _build_meta_model_for_fsdp(cls, model_path, kwargs):
+        config = kwargs.get("config")
+        if config is None:
+            config = AutoConfig.from_pretrained(
+                model_path,
+                trust_remote_code=kwargs.get("trust_remote_code", True),
+            )
+
+        init_kwargs = {}
+        torch_dtype = kwargs.get("torch_dtype", None)
+        if torch_dtype is not None:
+            init_kwargs["torch_dtype"] = torch_dtype
+        if "attn_implementation" in kwargs:
+            init_kwargs["attn_implementation"] = kwargs["attn_implementation"]
+        if "experts_implementation" in kwargs:
+            init_kwargs["experts_implementation"] = kwargs["experts_implementation"]
+
+        with torch.device("meta"):
+            model = cls._from_config(config, **init_kwargs)
+        return _disable_router_logits_if_needed(model)
+
+    def _fsdp_safe_load(cls, pretrained_model_name_or_path, *args, **kwargs):
+        kwargs = dict(kwargs)
+        kwargs.setdefault("low_cpu_mem_usage", True)
+
+        local_rank = int(os.environ.get("LOCAL_RANK", "0"))
+        if local_rank != 0:
+            logger.info(
+                "[HYV4 Patch 3] FSDP mode: local_rank=%d != 0, creating model on meta device. "
+                "Weights will be synchronized from rank 0.",
+                local_rank,
+            )
+            return _build_meta_model_for_fsdp(cls, pretrained_model_name_or_path, kwargs)
+
+        logger.info(
+            "[HYV4 Patch 3] FSDP mode: local_rank=0, loading real weights to CPU "
+            "with low_cpu_mem_usage enabled."
+        )
+        kwargs["device_map"] = {"": "cpu"}
+        model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
+        return _disable_router_logits_if_needed(model)
+
+    def _ensure_zero3_config_and_load(cls, pretrained_model_name_or_path, *args, **kwargs):
+        """Ensure HfDeepSpeedConfig is set before calling from_pretrained."""
+        model_path = pretrained_model_name_or_path
+        print(f"[HYV4 Patch 3] _ensure_zero3_config_and_load called with path: {model_path}", flush=True)
+
+        if not (isinstance(model_path, str) and os.path.isdir(model_path)):
+            return _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
+
+        index_file = os.path.join(model_path, "model.safetensors.index.json")
+        single_file = os.path.join(model_path, "model.safetensors")
+        if not (os.path.isfile(index_file) or os.path.isfile(single_file)):
+            return _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
+
+        if _is_fsdp_requested():
+            return _fsdp_safe_load(cls, pretrained_model_name_or_path, *args, **kwargs)
+
+        from transformers.integrations.deepspeed import is_deepspeed_zero3_enabled
+
+        if is_deepspeed_zero3_enabled():
+            print("[HYV4 Patch 3] ZeRO-3 already enabled, using native from_pretrained.", flush=True)
+            model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
+            return _disable_router_logits_if_needed(model)
+
+        ds_config_path = os.environ.get("DEEPSPEED_CONFIG_FILE", None)
+        if ds_config_path is None:
+            ds_config_path = os.environ.get("DEEPSPEED_CONFIG", None)
+
+        if ds_config_path is None:
+            for i, arg in enumerate(sys.argv):
+                if arg == '--deepspeed' and i + 1 < len(sys.argv):
+                    ds_config_path = sys.argv[i + 1]
+                    break
+
+        if ds_config_path is None or not os.path.isfile(ds_config_path):
+            print("[HYV4 Patch 3] No DeepSpeed config found, using default from_pretrained.", flush=True)
+            model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
+            return _disable_router_logits_if_needed(model)
+
+        with open(ds_config_path, "r") as f:
+            ds_config = _json.load(f)
+
+        zero_stage = ds_config.get("zero_optimization", {}).get("stage", 0)
+        if zero_stage != 3:
+            print(f"[HYV4 Patch 3] Not ZeRO-3 (stage={zero_stage}), using default.", flush=True)
+            model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
+            return _disable_router_logits_if_needed(model)
+
+        print(f"[HYV4 Patch 3] Setting HfDeepSpeedConfig for ZeRO-3 native loading: {ds_config_path}", flush=True)
+
+        from transformers.integrations.deepspeed import HfDeepSpeedConfig
+        _ds_config_obj = HfDeepSpeedConfig(ds_config_path)
+
+        model = _real_orig_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs)
+
+        print("[HYV4 Patch 3] Native ZeRO-3 from_pretrained completed.", flush=True)
+        return _disable_router_logits_if_needed(model)
+
+    @classmethod
+    def _patched_from_pretrained(cls, pretrained_model_name_or_path, *args, **kwargs):
+        return _ensure_zero3_config_and_load(cls, pretrained_model_name_or_path, *args, **kwargs)
+
+    PreTrainedModel.from_pretrained = _patched_from_pretrained
+
+    logger.info("[HYV4 Patch 3] Loading patch applied for ZeRO-3 and FSDP1.")
+
+
+# ============================================================================
+# Optional Patch (NOT auto-applied): LoRA z3_leaf marking
+#
+# PEFT wraps target Linear layers with lora.Linear, adding extra sub-modules
+# (base_layer, lora_A, lora_B). This changes the module tree structure and
+# disrupts ZeRO-3's parameter fetch/release scheduling, causing OOM during
+# backward recomputation. By marking these wrappers as z3_leaf, ZeRO-3 treats
+# them as atomic units, restoring correct scheduling.
+#
+# This is NOT auto-applied because it requires the model to have LoRA already
+# applied. Call apply_lora_z3_leaf_patch(model) manually after LoRA setup.
+# ============================================================================
+
+def apply_lora_z3_leaf_patch(model):
+    """Mark PEFT LoRA wrapper modules as ZeRO-3 leaf modules.
+
+    This is an OPTIONAL patch. Call manually AFTER LoRA has been applied
+    to the model and BEFORE training starts.
+    """
+    try:
+        from deepspeed.utils import set_z3_leaf_module
+        from peft.tuners.lora import Linear as LoraLinear
+    except ImportError:
+        logger.info("[HYV4 Optional] DeepSpeed or PEFT not available, skipping z3_leaf patch.")
+        return
+
+    z3_leaf_count = 0
+    for module in model.modules():
+        if isinstance(module, LoraLinear):
+            set_z3_leaf_module(module, True)
+            z3_leaf_count += 1
+
+    logger.info("[HYV4 Optional] Marked %d LoraLinear modules with _z3_leaf=True.", z3_leaf_count)
+
+
+# ============================================================================
+# Patch 4: Fix logging_dir compatibility between ms-swift and
+# transformers 5.x
+#
+# ms-swift's SftArguments._add_version() accesses self.logging_dir,
+# expecting it to be inherited from transformers.TrainingArguments as a
+# dataclass field. However, in transformers 5.x, logging_dir has been
+# deprecated and is no longer included in dataclass fields (init=False or
+# removed from __dataclass_fields__). This causes AttributeError.
+#
+# Fix: Monkey-patch _add_version to ensure logging_dir exists before access.
+# ============================================================================
+
+def _apply_logging_dir_patch():
+    """Fix SftArguments._add_version for transformers 5.x compatibility."""
+    try:
+        from swift.arguments.sft_args import SftArguments
+
+        _orig_add_version = SftArguments._add_version
+
+        def _patched_add_version(self):
+            # Ensure logging_dir attribute exists (transformers 5.x removed it
+            # from dataclass fields but ms-swift 4.4.2 still accesses it)
+            if not hasattr(self, 'logging_dir'):
+                self.logging_dir = None
+            # Also ensure run_name exists (may also be affected)
+            if not hasattr(self, 'run_name'):
+                self.run_name = None
+            _orig_add_version(self)
+
+        SftArguments._add_version = _patched_add_version
+        logger.info(
+            "[HYV4 Patch 4] Patched SftArguments._add_version for "
+            "transformers 5.x logging_dir compatibility."
+        )
+    except (ImportError, AttributeError) as e:
+        logger.info("[HYV4 Patch 4] Could not apply logging_dir patch: %s", e)
+
+
+# ==========================================================================
+# Patch 5: Mirror LLaMA Factory's FSDP1 dtype + mixed precision guard
+#
+# After LoRA injection, adapter weights can stay in fp32 while the base model
+# is bf16. In addition, Accelerate may re-enable mixed precision policies that
+# upcast bf16 shards back to fp32 during FSDP wrapping. Both behaviors increase
+# memory pressure significantly.
+#
+# We align ms-swift with the previously successful LLaMA Factory setup by:
+#   1. unifying floating-point parameters to bf16 before FSDP wrap
+#   2. disabling Accelerate mixed precision for the FSDP plugin
+# ============================================================================
+
+def _apply_fsdp_dtype_patch():
+    """Unify parameter dtype to bf16 and disable Accelerate FSDP1 mixed precision."""
+    try:
+        from transformers import Trainer
+
+        _orig_prepare_for_training = Trainer._prepare_for_training
+
+        def _patched_prepare_for_training(self, *args, **kwargs):
+            if getattr(self.args, 'fsdp', False):
+                dtype_counts = {}
+                for p in self.model.parameters():
+                    dt = str(p.dtype)
+                    dtype_counts[dt] = dtype_counts.get(dt, 0) + 1
+
+                if len(dtype_counts) > 1:
+                    logger.info(
+                        "[HYV4 Patch 5] Mixed dtypes detected before FSDP wrap: %s. "
+                        "Casting floating-point parameters to bfloat16.",
+                        dtype_counts,
+                    )
+                    for p in self.model.parameters():
+                        if p.dtype != torch.bfloat16 and p.dtype.is_floating_point:
+                            p.data = p.data.to(torch.bfloat16)
+                else:
+                    logger.info("[HYV4 Patch 5] Parameter dtypes already uniform: %s", dtype_counts)
+
+                try:
+                    if hasattr(self, 'accelerator') and hasattr(self.accelerator, 'state'):
+                        old_mp = getattr(self.accelerator.state, '_mixed_precision', None)
+                        self.accelerator.state._mixed_precision = 'no'
+                        fsdp_plugin = getattr(self.accelerator.state, 'fsdp_plugin', None)
+                        if fsdp_plugin is not None:
+                            if hasattr(fsdp_plugin, 'mixed_precision_policy'):
+                                fsdp_plugin.mixed_precision_policy = None
+                            if hasattr(fsdp_plugin, 'kwargs') and isinstance(fsdp_plugin.kwargs, dict):
+                                fsdp_plugin.kwargs.pop('mixed_precision', None)
+                        logger.info(
+                            "[HYV4 Patch 5] Disabled Accelerate FSDP1 mixed precision "
+                            "(previous state: %s).",
+                            old_mp,
+                        )
+                    os.environ['ACCELERATE_MIXED_PRECISION'] = 'no'
+                except Exception as e:
+                    logger.warning("[HYV4 Patch 5] Failed to disable mixed precision: %s", e)
+
+            return _orig_prepare_for_training(self, *args, **kwargs)
+
+        Trainer._prepare_for_training = _patched_prepare_for_training
+        logger.info("[HYV4 Patch 5] FSDP1 dtype + mixed precision guard applied.")
+    except (ImportError, AttributeError) as e:
+        logger.info("[HYV4 Patch 5] Could not apply FSDP1 dtype patch: %s", e)
+
+
+# ============================================================================
+# Patch 6: Disable _compute_acc during training
+#
+# ms-swift computes training accuracy (argmax on logits) at every step.
+# This requires keeping outputs.logits in memory until acc is computed,
+# adding extra GPU/CPU memory pressure. LLaMA Factory and DeepSpeed native
+# scripts do NOT compute training accuracy. Disabling this aligns ms-swift
+# with the other frameworks and reduces memory usage during forward pass.
+# ============================================================================
+
+def _apply_disable_compute_acc_patch():
+    """Patch _compute_acc to be a no-op during training."""
+    try:
+        from swift.trainers.mixin import SwiftMixin
+
+        def _noop_compute_acc(self, outputs, labels, cu_seqlens=None):
+            return
+
+        SwiftMixin._compute_acc = _noop_compute_acc
+        print("[HYV4 Patch 6] Disabled _compute_acc to reduce memory usage.", flush=True)
+    except (ImportError, AttributeError) as e:
+        print(f"[HYV4 Patch 6] Could not apply _compute_acc patch: {e}", flush=True)
+
+
+# ============================================================================
+# Auto-apply patches on import
+# ============================================================================
+
+# Patch 2: Skip grad norm (always safe to apply; no-op if DeepSpeed not used)
+_apply_skip_grad_norm_patch()
+
+# Patch 3: Memory-efficient model loading
+_apply_shard_loading_patch()
+
+# Patch 4: Fix logging_dir compatibility
+_apply_logging_dir_patch()
+
+# Patch 5: Align FSDP1 dtype / mixed precision behavior with LLaMA Factory
+_apply_fsdp_dtype_patch()
+
+# Patch 6: Disable _compute_acc
+_apply_disable_compute_acc_patch()
+
+logger.info("[HYV4] All ms-swift patches loaded successfully.")

finetune/ms_swift_support/sft_train.sh

@@ -0,0 +1,101 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# -------------------- Network Configuration --------------------
+NET_TYPE="high"
+export NCCL_DEBUG=WARN
+export NCCL_P2P_LEVEL=NVL
+export NCCL_IB_TIMEOUT=24
+export NCCL_NVLS_ENABLE=0
+export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
+if [[ "${NET_TYPE}" = "low" ]]; then
+    export NCCL_SOCKET_IFNAME=eth1
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_HCA=mlx5_2:1
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+else
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_IB_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+    export NCCL_SOCKET_IFNAME=bond1
+    export UCX_NET_DEVICES=bond1
+    export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
+    export NCCL_COLLNET_ENABLE=0
+    export SHARP_COLL_ENABLE_SAT=0
+    export NCCL_NET_GDR_LEVEL=2
+    export NCCL_IB_QPS_PER_CONNECTION=4
+    export NCCL_IB_TC=160
+    export NCCL_PXN_DISABLE=1
+fi
+
+# -------------------- Node Configuration --------------------
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
+export IP_LIST=${IP_LIST:-"127.0.0.1"}
+
+MASTER_PORT=${MASTER_PORT:-29500}
+
+IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
+NODES=${#IP_ARRAY[@]}
+MASTER_ADDR=${IP_ARRAY[0]}
+
+# -------------------- Paths --------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+YAML_FILE="${YAML_FILE:-hy_v4_full_sft.yaml}"
+PATCHES_FILE="${SCRIPT_DIR}/hy_v4_swift_patches.py"
+CONFIG_FILE="${SCRIPT_DIR}/${YAML_FILE}"
+
+# -------------------- Distributed Environment --------------------
+export MASTER_ADDR="${MASTER_ADDR}"
+export MASTER_PORT="${MASTER_PORT}"
+export NNODES="${NODES}"
+
+if [ ${NODES} -gt 1 ]; then
+    # Determine local node rank by matching local IP against IP_LIST
+    LOCAL_IP=$(hostname -i | awk '{print $1}')
+    NODE_RANK=0
+    for i in "${!IP_ARRAY[@]}"; do
+        if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
+            NODE_RANK=$i
+            break
+        fi
+    done
+    export RANK="${NODE_RANK}"
+else
+    export RANK=0
+fi
+
+echo "============================================"
+echo "  HYV4 ms-swift SFT Training (Full + DeepSpeed)"
+echo "  Nodes: ${NNODES}, Rank: ${RANK}"
+echo "  Master: ${MASTER_ADDR}:${MASTER_PORT}"
+echo "  GPUs per node: ${HOST_GPU_NUM}"
+echo "  Total GPUs: $((NODES * HOST_GPU_NUM))"
+echo "  Config: ${CONFIG_FILE}"
+echo "============================================"
+
+# -------------------- Launch --------------------
+# Add script directory to PYTHONPATH so patches can find local modules
+export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${SCRIPT_DIR}"
+
+torchrun \
+    --nnodes "${NNODES}" \
+    --node_rank "${RANK}" \
+    --nproc_per_node "${HOST_GPU_NUM}" \
+    --master_addr "${MASTER_ADDR}" \
+    --master_port "${MASTER_PORT}" \
+    -m swift.cli.sft \
+    --custom_register_path "${PATCHES_FILE}" \
+    --config "${CONFIG_FILE}"

finetune/ms_swift_support/sft_train_lora.sh

@@ -0,0 +1,104 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# -------------------- Network Configuration --------------------
+NET_TYPE="high"
+export NCCL_DEBUG=WARN
+export NCCL_P2P_LEVEL=NVL
+export NCCL_IB_TIMEOUT=24
+export NCCL_NVLS_ENABLE=0
+export NCCL_MPI_PROFILE_PRIMS_ENABLE=0
+export CUDA_DEVICE_MAX_CONNECTIONS=1
+export TORCH_NCCL_HEARTBEAT_TIMEOUT_SEC=3600
+export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:128
+if [[ "${NET_TYPE}" = "low" ]]; then
+    export NCCL_SOCKET_IFNAME=eth1
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_HCA=mlx5_2:1
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+else
+    export NCCL_IB_GID_INDEX=3
+    export NCCL_IB_SL=3
+    export NCCL_CHECK_DISABLE=1
+    export NCCL_P2P_DISABLE=0
+    export NCCL_IB_DISABLE=0
+    export NCCL_LL_THRESHOLD=16384
+    export NCCL_IB_CUDA_SUPPORT=1
+    export NCCL_SOCKET_IFNAME=bond1
+    export UCX_NET_DEVICES=bond1
+    export NCCL_IB_HCA=mlx5_bond_1,mlx5_bond_5,mlx5_bond_3,mlx5_bond_7,mlx5_bond_4,mlx5_bond_8,mlx5_bond_2,mlx5_bond_6
+    export NCCL_COLLNET_ENABLE=0
+    export SHARP_COLL_ENABLE_SAT=0
+    export NCCL_NET_GDR_LEVEL=2
+    export NCCL_IB_QPS_PER_CONNECTION=4
+    export NCCL_IB_TC=160
+    export NCCL_PXN_DISABLE=1
+fi
+
+# -------------------- Node Configuration --------------------
+export HOST_GPU_NUM=8
+# IP list, comma separated. e.g. "10.0.0.1,10.0.0.2" or single node "127.0.0.1"
+export IP_LIST=${IP_LIST:-"127.0.0.1"}
+
+MASTER_PORT=${MASTER_PORT:-29500}
+
+IFS=',' read -ra IP_ARRAY <<< "$IP_LIST"
+NODES=${#IP_ARRAY[@]}
+MASTER_ADDR=${IP_ARRAY[0]}
+
+# -------------------- Paths --------------------
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+YAML_FILE="${YAML_FILE:-hy_v4_lora_sft.yaml}"
+PATCHES_FILE="${SCRIPT_DIR}/hy_v4_swift_patches.py"
+CONFIG_FILE="${SCRIPT_DIR}/${YAML_FILE}"
+
+# -------------------- Distributed Environment --------------------
+export MASTER_ADDR="${MASTER_ADDR}"
+export MASTER_PORT="${MASTER_PORT}"
+export NNODES="${NODES}"
+
+# Prevent Accelerate/FSDP from upcasting bf16 parameters back to fp32
+export ACCELERATE_MIXED_PRECISION=no
+
+if [ ${NODES} -gt 1 ]; then
+    # Determine local node rank by matching local IP against IP_LIST
+    LOCAL_IP=$(hostname -i | awk '{print $1}')
+    NODE_RANK=0
+    for i in "${!IP_ARRAY[@]}"; do
+        if [[ "${IP_ARRAY[$i]}" == "${LOCAL_IP}" ]]; then
+            NODE_RANK=$i
+            break
+        fi
+    done
+    export RANK="${NODE_RANK}"
+else
+    export RANK=0
+fi
+
+echo "============================================"
+echo "  HYV4 ms-swift SFT Training (LoRA + FSDP)"
+echo "  Nodes: ${NNODES}, Rank: ${RANK}"
+echo "  Master: ${MASTER_ADDR}:${MASTER_PORT}"
+echo "  GPUs per node: ${HOST_GPU_NUM}"
+echo "  Total GPUs: $((NODES * HOST_GPU_NUM))"
+echo "  Config: ${CONFIG_FILE}"
+echo "============================================"
+
+# -------------------- Launch --------------------
+# Add script directory to PYTHONPATH so patches can find local modules
+export PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${SCRIPT_DIR}"
+
+torchrun \
+    --nnodes "${NNODES}" \
+    --node_rank "${RANK}" \
+    --nproc_per_node "${HOST_GPU_NUM}" \
+    --master_addr "${MASTER_ADDR}" \
+    --master_port "${MASTER_PORT}" \
+    -m swift.cli.sft \
+    --custom_register_path "${PATCHES_FILE}" \
+    --config "${CONFIG_FILE}"

finetune/requirements.txt

@@ -0,0 +1,12 @@
+transformers>=5.16.2
+torch>=2.10.0
+torchvision
+torchaudio
+accelerate>=1.11.0
+peft>=0.18.1
+deepspeed>=0.18.7
+sentencepiece
+protobuf
+ninja
+flash-attn
+tensorboard