diff --git a/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/global_step7000/mp_rank_00_model_states.pt b/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/global_step7000/mp_rank_00_model_states.pt new file mode 100644 index 0000000000000000000000000000000000000000..ea80e4cd1b2e4fc6535f0c32ab3195b41079ba9f --- /dev/null +++ b/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/global_step7000/mp_rank_00_model_states.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c42afe0d84d6312de0c4d2bb7302c5d9db244998ba1df02858f948affbce9249 +size 10261699974 diff --git a/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/latest b/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/latest new file mode 100644 index 0000000000000000000000000000000000000000..02f8b21f7d75fb06ab0c850d040bab5a1b2af782 --- /dev/null +++ b/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/latest @@ -0,0 +1 @@ +global_step7000 \ No newline at end of file diff --git a/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/zero_to_fp32.py b/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/zero_to_fp32.py new file mode 100644 index 0000000000000000000000000000000000000000..24cc342e78d1a006c782b3a4cd68d9ce786d8fd8 --- /dev/null +++ b/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/zero_to_fp32.py @@ -0,0 +1,604 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: python zero_to_fp32.py . pytorch_model.bin + +import argparse +import torch +import glob +import math +import os +import re +from collections import OrderedDict +from dataclasses import dataclass + +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS, + FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES, + FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS) + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + + total_files = len(files) + state_dicts = [] + for f in files: + state_dict = torch.load(f, map_location=device) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + if zero_stage <= 2: + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + elif zero_stage == 3: + # if there is more than one param group, there will be multiple flattened tensors - one + # flattened tensor per group - for simplicity merge them into a single tensor + # + # XXX: could make the script more memory efficient for when there are multiple groups - it + # will require matching the sub-lists of param_shapes for each param group flattened tensor + + fp32_flat_groups = [ + torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts)) + ] + + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = fp32_flat_groups[0].numel() * world_size + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + for name, shape in param_shapes.items(): + + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # XXX: memory usage doubles here + state_dict[name] = torch.cat( + tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)), + 0).narrow(0, 0, unpartitioned_numel).view(shape) + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + + Returns: + - pytorch ``state_dict`` + + Note: this approach may not work if your application doesn't have sufficient free CPU memory and + you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + + +def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters) + print(f"Saving fp32 state dict to {output_file}") + torch.save(state_dict, output_file) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument( + "output_file", + type=str, + help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir, + args.output_file, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/log.txt b/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/log.txt new file mode 100644 index 0000000000000000000000000000000000000000..d78e7fd79a6f475b2fa3aab2be22aaf7fdc26e15 --- /dev/null +++ b/ckpts/qwen3-1.7b-whisper-260205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/log.txt @@ -0,0 +1,32384 @@ +[2026-02-06 15:54:46,504] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2026-02-06 15:54:47,905] [WARNING] [runner.py:202:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only. +[2026-02-06 15:54:47,906] [INFO] [runner.py:568:main] cmd = /usr/bin/python -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMCwgMSwgMiwgM119 --master_addr=127.0.0.1 --master_port=29500 --enable_each_rank_log=None pretrain_demo_qwenaudio.py --num_worker 16 --save_path /fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked --save_steps 7000 --logging_steps 1 --eval_steps 4000001 --train_batch_size 80 --micro_train_batch_size 10 --micro_eval_batch_size 8 --model_path /fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-1227_4x3000h_multistage/ckpts/global_step34000_hf_merged_lora --tokenizer_path /fs/nlp/common/plms/qwen-audio/qwen3-1.7b-chat-audio-whisper-v3-convproj --max_epochs 3 --max_len 1024 --zero_stage 2 --max_ckpt_num 5 --learning_rate 2e-5 --flash_attn --use_custom_qwen3 --dataset_config /fs/nlp/common_intern/meiyuxiang/uniscale_multimodal/scripts/myx/data_config/train_config_12x1000h_mls.yaml --tensorboard-dir /fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/tb_logs --n_mels 128 --audio_subsampling_scale 4 --packing_samples --seed 421 --freeze_llm --freeze_audio --lora_rank 32 --lora_alpha 64 --specaug_policy SM --post_audio_ratio 1 --gradient_checkpointing --load_checkpoint --bf16 --use_zipper_lora --use_lid_router --zipper_lora_num_languages 12 --zipper_lora_r 32 --trainable_A --zipper_lora_scope audio --zipper_lora_use_independent +[2026-02-06 15:54:50,123] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2026-02-06 15:54:51,405] [INFO] [launch.py:138:main] 0 NCCL_VERSION=2.19.3 +[2026-02-06 15:54:51,405] [INFO] [launch.py:138:main] 0 NCCL_SOCKET_IFNAME=eth1 +[2026-02-06 15:54:51,405] [INFO] [launch.py:138:main] 0 NCCL_CUMEM_HOST_ENABLE=0 +[2026-02-06 15:54:51,405] [INFO] [launch.py:138:main] 0 NCCL_IB_HCA=^=mlx5_4 +[2026-02-06 15:54:51,405] [INFO] [launch.py:145:main] WORLD INFO DICT: {'localhost': [0, 1, 2, 3]} +[2026-02-06 15:54:51,405] [INFO] [launch.py:151:main] nnodes=1, num_local_procs=4, node_rank=0 +[2026-02-06 15:54:51,405] [INFO] [launch.py:162:main] global_rank_mapping=defaultdict(, {'localhost': [0, 1, 2, 3]}) +[2026-02-06 15:54:51,405] [INFO] [launch.py:163:main] dist_world_size=4 +[2026-02-06 15:54:51,405] [INFO] [launch.py:165:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3 +[2026-02-06 15:54:51,406] [INFO] [launch.py:253:main] process 707055 spawned with command: ['/usr/bin/python', '-u', 'pretrain_demo_qwenaudio.py', '--local_rank=0', '--num_worker', '16', '--save_path', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked', '--save_steps', '7000', '--logging_steps', '1', '--eval_steps', '4000001', '--train_batch_size', '80', '--micro_train_batch_size', '10', '--micro_eval_batch_size', '8', '--model_path', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-1227_4x3000h_multistage/ckpts/global_step34000_hf_merged_lora', '--tokenizer_path', '/fs/nlp/common/plms/qwen-audio/qwen3-1.7b-chat-audio-whisper-v3-convproj', '--max_epochs', '3', '--max_len', '1024', '--zero_stage', '2', '--max_ckpt_num', '5', '--learning_rate', '2e-5', '--flash_attn', '--use_custom_qwen3', '--dataset_config', '/fs/nlp/common_intern/meiyuxiang/uniscale_multimodal/scripts/myx/data_config/train_config_12x1000h_mls.yaml', '--tensorboard-dir', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/tb_logs', '--n_mels', '128', '--audio_subsampling_scale', '4', '--packing_samples', '--seed', '421', '--freeze_llm', '--freeze_audio', '--lora_rank', '32', '--lora_alpha', '64', '--specaug_policy', 'SM', '--post_audio_ratio', '1', '--gradient_checkpointing', '--load_checkpoint', '--bf16', '--use_zipper_lora', '--use_lid_router', '--zipper_lora_num_languages', '12', '--zipper_lora_r', '32', '--trainable_A', '--zipper_lora_scope', 'audio', '--zipper_lora_use_independent'] +[2026-02-06 15:54:51,407] [INFO] [launch.py:253:main] process 707056 spawned with command: ['/usr/bin/python', '-u', 'pretrain_demo_qwenaudio.py', '--local_rank=1', '--num_worker', '16', '--save_path', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked', '--save_steps', '7000', '--logging_steps', '1', '--eval_steps', '4000001', '--train_batch_size', '80', '--micro_train_batch_size', '10', '--micro_eval_batch_size', '8', '--model_path', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-1227_4x3000h_multistage/ckpts/global_step34000_hf_merged_lora', '--tokenizer_path', '/fs/nlp/common/plms/qwen-audio/qwen3-1.7b-chat-audio-whisper-v3-convproj', '--max_epochs', '3', '--max_len', '1024', '--zero_stage', '2', '--max_ckpt_num', '5', '--learning_rate', '2e-5', '--flash_attn', '--use_custom_qwen3', '--dataset_config', '/fs/nlp/common_intern/meiyuxiang/uniscale_multimodal/scripts/myx/data_config/train_config_12x1000h_mls.yaml', '--tensorboard-dir', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/tb_logs', '--n_mels', '128', '--audio_subsampling_scale', '4', '--packing_samples', '--seed', '421', '--freeze_llm', '--freeze_audio', '--lora_rank', '32', '--lora_alpha', '64', '--specaug_policy', 'SM', '--post_audio_ratio', '1', '--gradient_checkpointing', '--load_checkpoint', '--bf16', '--use_zipper_lora', '--use_lid_router', '--zipper_lora_num_languages', '12', '--zipper_lora_r', '32', '--trainable_A', '--zipper_lora_scope', 'audio', '--zipper_lora_use_independent'] +[2026-02-06 15:54:51,407] [INFO] [launch.py:253:main] process 707057 spawned with command: ['/usr/bin/python', '-u', 'pretrain_demo_qwenaudio.py', '--local_rank=2', '--num_worker', '16', '--save_path', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked', '--save_steps', '7000', '--logging_steps', '1', '--eval_steps', '4000001', '--train_batch_size', '80', '--micro_train_batch_size', '10', '--micro_eval_batch_size', '8', '--model_path', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-1227_4x3000h_multistage/ckpts/global_step34000_hf_merged_lora', '--tokenizer_path', '/fs/nlp/common/plms/qwen-audio/qwen3-1.7b-chat-audio-whisper-v3-convproj', '--max_epochs', '3', '--max_len', '1024', '--zero_stage', '2', '--max_ckpt_num', '5', '--learning_rate', '2e-5', '--flash_attn', '--use_custom_qwen3', '--dataset_config', '/fs/nlp/common_intern/meiyuxiang/uniscale_multimodal/scripts/myx/data_config/train_config_12x1000h_mls.yaml', '--tensorboard-dir', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/tb_logs', '--n_mels', '128', '--audio_subsampling_scale', '4', '--packing_samples', '--seed', '421', '--freeze_llm', '--freeze_audio', '--lora_rank', '32', '--lora_alpha', '64', '--specaug_policy', 'SM', '--post_audio_ratio', '1', '--gradient_checkpointing', '--load_checkpoint', '--bf16', '--use_zipper_lora', '--use_lid_router', '--zipper_lora_num_languages', '12', '--zipper_lora_r', '32', '--trainable_A', '--zipper_lora_scope', 'audio', '--zipper_lora_use_independent'] +[2026-02-06 15:54:51,408] [INFO] [launch.py:253:main] process 707058 spawned with command: ['/usr/bin/python', '-u', 'pretrain_demo_qwenaudio.py', '--local_rank=3', '--num_worker', '16', '--save_path', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked', '--save_steps', '7000', '--logging_steps', '1', '--eval_steps', '4000001', '--train_batch_size', '80', '--micro_train_batch_size', '10', '--micro_eval_batch_size', '8', '--model_path', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-1227_4x3000h_multistage/ckpts/global_step34000_hf_merged_lora', '--tokenizer_path', '/fs/nlp/common/plms/qwen-audio/qwen3-1.7b-chat-audio-whisper-v3-convproj', '--max_epochs', '3', '--max_len', '1024', '--zero_stage', '2', '--max_ckpt_num', '5', '--learning_rate', '2e-5', '--flash_attn', '--use_custom_qwen3', '--dataset_config', '/fs/nlp/common_intern/meiyuxiang/uniscale_multimodal/scripts/myx/data_config/train_config_12x1000h_mls.yaml', '--tensorboard-dir', '/fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/tb_logs', '--n_mels', '128', '--audio_subsampling_scale', '4', '--packing_samples', '--seed', '421', '--freeze_llm', '--freeze_audio', '--lora_rank', '32', '--lora_alpha', '64', '--specaug_policy', 'SM', '--post_audio_ratio', '1', '--gradient_checkpointing', '--load_checkpoint', '--bf16', '--use_zipper_lora', '--use_lid_router', '--zipper_lora_num_languages', '12', '--zipper_lora_r', '32', '--trainable_A', '--zipper_lora_scope', 'audio', '--zipper_lora_use_independent'] +[2026-02-06 15:54:55,747] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2026-02-06 15:54:55,749] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2026-02-06 15:54:55,769] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2026-02-06 15:54:55,807] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect) +[2026-02-06 15:54:56,142] [INFO] [comm.py:637:init_distributed] cdb=None +[2026-02-06 15:54:56,142] [INFO] [comm.py:668:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl +[2026-02-06 15:54:56,294] [INFO] [comm.py:637:init_distributed] cdb=None +[2026-02-06 15:54:56,299] [INFO] [comm.py:637:init_distributed] cdb=None +[W CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator()) +[W CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator()) +[2026-02-06 15:54:56,313] [INFO] [comm.py:637:init_distributed] cdb=None +[W CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator()) +[W CUDAAllocatorConfig.h:28] Warning: expandable_segments not supported on this platform (function operator()) +2026-02-06 15:54:56.826 | INFO | model.unigpt_audio_models.tokenization_qwen_audio:__init__:147 - vocab_size: 155165, audio_start_id: 155163, audio_end_id: 155164, audio_pad_id: 151851. +2026-02-06 15:54:56.837 | INFO | model.unigpt_audio_models.tokenization_qwen_audio:__init__:147 - vocab_size: 155165, audio_start_id: 155163, audio_end_id: 155164, audio_pad_id: 151851. +2026-02-06 15:54:56.852 | INFO | model.unigpt_audio_models.tokenization_qwen_audio:__init__:147 - vocab_size: 155165, audio_start_id: 155163, audio_end_id: 155164, audio_pad_id: 151851. +loading dataset com_voice_ar with + 0it [00:00, ?it/s]loading dataset com_voice_ar with + 0it [00:00, ?it/s]2026-02-06 15:54:56.882 | INFO | model.unigpt_audio_models.tokenization_qwen_audio:__init__:147 - vocab_size: 155165, audio_start_id: 155163, audio_end_id: 155164, audio_pad_id: 151851. +loading dataset com_voice_ar with + 0it [00:00, ?it/s]loading dataset com_voice_ar with + 0it [00:00, ?it/s] 3842it [00:00, 38411.06it/s] 4013it [00:00, 40125.73it/s] 4113it [00:00, 41126.12it/s] 4034it [00:00, 40331.96it/s] 6363it [00:00, 39199.68it/s] +2026-02-06 15:54:57.025 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1272 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ar_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} + 0it [00:00, ?it/s] 6363it [00:00, 40098.20it/s] +2026-02-06 15:54:57.030 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1272 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ar_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} + 0it [00:00, ?it/s] 6363it [00:00, 41130.26it/s] +2026-02-06 15:54:57.040 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1272 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ar_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} + 0it [00:00, ?it/s] 6363it [00:00, 40521.02it/s] +2026-02-06 15:54:57.074 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1272 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ar_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} + 0it [00:00, ?it/s] 4790it [00:00, 47893.77it/s] 4790it [00:00, 47894.34it/s] 4872it [00:00, 48713.43it/s] 6363it [00:00, 44400.52it/s] +2026-02-06 15:54:57.170 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 6363 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ar_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_de with + 0it [00:00, ?it/s] 6363it [00:00, 45730.01it/s] +2026-02-06 15:54:57.171 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 6363 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ar_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_de with + 0it [00:00, ?it/s] 4765it [00:00, 47643.34it/s] 6363it [00:00, 44804.38it/s] +2026-02-06 15:54:57.184 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 6363 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ar_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_de with + 0it [00:00, ?it/s] 6363it [00:00, 43686.88it/s] +2026-02-06 15:54:57.221 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 6363 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ar_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_de with + 0it [00:00, ?it/s] 12054it [00:00, 120532.64it/s] 11938it [00:00, 119369.87it/s] 13264it [00:00, 132635.07it/s] 12219it [00:00, 122181.96it/s] 25080it [00:00, 126250.30it/s] 24644it [00:00, 123886.44it/s] 26528it [00:00, 132138.25it/s] 25041it [00:00, 125726.67it/s] 38286it [00:00, 128898.33it/s] 37722it [00:00, 127026.06it/s] 39875it [00:00, 132739.82it/s] 38166it [00:00, 128245.61it/s] 50425it [00:00, 126782.21it/s] 51176it [00:00, 127893.43it/s] 53150it [00:00, 130858.69it/s] 50991it [00:00, 127251.50it/s] 63529it [00:00, 131284.08it/s] 63529it [00:00, 127568.24it/s] + + 63318it [00:00, 127551.39it/s] 63529it [00:00, 126492.59it/s] +2026-02-06 15:54:57.690 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 32304 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_de_pack_new.jsonl +2026-02-06 15:54:57.691 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 32304 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_de_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_es with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_es with +2026-02-06 15:54:57.696 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 32304 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_de_pack_new.jsonl + 0it [00:00, ?it/s] 0it [00:00, ?it/s]{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_es with + 0it [00:00, ?it/s] 63529it [00:00, 127999.71it/s] +2026-02-06 15:54:57.742 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 32304 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_de_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_es with + 0it [00:00, ?it/s] 11831it [00:00, 118293.19it/s] 11890it [00:00, 118892.18it/s] 12224it [00:00, 122225.55it/s] 11982it [00:00, 119810.69it/s] 24781it [00:00, 124879.51it/s] 24841it [00:00, 125126.10it/s] 25096it [00:00, 126041.46it/s] 30551it [00:00, 122394.96it/s] 30551it [00:00, 122618.78it/s] + + 24902it [00:00, 125326.32it/s] 30551it [00:00, 123978.23it/s] +2026-02-06 15:54:57.952 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 33300 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_es_pack_new.jsonl +2026-02-06 15:54:57.952 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 33300 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_es_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} + 0it [00:00, ?it/s] 0it [00:00, ?it/s]2026-02-06 15:54:57.955 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 33300 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_es_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} + 0it [00:00, ?it/s] 30551it [00:00, 123154.99it/s] +2026-02-06 15:54:58.004 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 33300 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_es_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} + 0it [00:00, ?it/s] 11431it [00:00, 114301.66it/s] 11436it [00:00, 114352.75it/s] 11471it [00:00, 114698.90it/s] 10879it [00:00, 108770.14it/s] 22862it [00:00, 114247.67it/s] 22872it [00:00, 114273.10it/s] 23000it [00:00, 115041.01it/s] 22331it [00:00, 112139.32it/s] 30551it [00:00, 114470.87it/s] 30551it [00:00, 114488.16it/s] + + 30551it [00:00, 115347.77it/s] +2026-02-06 15:54:58.221 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 30551 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_es_pack_new.jsonl +2026-02-06 15:54:58.221 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 30551 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_es_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11}{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} + +loading dataset mls_fr with loading dataset mls_fr with + + 0it [00:00, ?it/s] 0it [00:00, ?it/s]2026-02-06 15:54:58.222 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 30551 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_es_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_fr with + 0it [00:00, ?it/s] 30551it [00:00, 113299.43it/s] +2026-02-06 15:54:58.276 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 30551 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_es_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_fr with + 0it [00:00, ?it/s] 8579it [00:00, 85787.22it/s] 8584it [00:00, 85832.51it/s] 8723it [00:00, 87214.28it/s] 9023it [00:00, 90224.49it/s] 17205it [00:00, 86055.47it/s] 17205it [00:00, 86048.09it/s] 17445it [00:00, 86638.27it/s] 18046it [00:00, 88740.46it/s] 26001it [00:00, 86917.97it/s] 26001it [00:00, 86919.99it/s] 26282it [00:00, 87419.71it/s] 27001it [00:00, 89103.44it/s] 35066it [00:00, 88389.73it/s] 35067it [00:00, 88388.91it/s] 35213it [00:00, 88160.18it/s] 36216it [00:00, 87592.66it/s] + 36216it [00:00, 87400.99it/s] 36216it [00:00, 87402.15it/s] + +2026-02-06 15:54:58.658 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 33637 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_fr_pack_new.jsonl +2026-02-06 15:54:58.658 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 33637 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_fr_pack_new.jsonl +2026-02-06 15:54:58.658 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 33637 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_fr_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_it with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_it with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_it with + 0it [00:00, ?it/s] 0it [00:00, ?it/s] 0it [00:00, ?it/s] 36216it [00:00, 90183.48it/s] +2026-02-06 15:54:58.700 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 33637 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_fr_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_it with + 0it [00:00, ?it/s] 8282it [00:00, 125481.25it/s] 8282it [00:00, 125765.64it/s] + + 8282it [00:00, 124997.21it/s] +2026-02-06 15:54:58.726 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 16564 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_it_pack_new.jsonl +2026-02-06 15:54:58.726 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 16564 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_it_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11}loading dataset com_voice_ja with + +loading dataset com_voice_ja with +2026-02-06 15:54:58.727 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 16564 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_it_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_ja with + 0it [00:00, ?it/s] 0it [00:00, ?it/s] 0it [00:00, ?it/s] 8282it [00:00, 142709.59it/s] +2026-02-06 15:54:58.761 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 16564 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_it_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_ja with + 0it [00:00, ?it/s] 4155it [00:00, 41545.58it/s] 4155it [00:00, 41542.61it/s] 4166it [00:00, 41655.27it/s] 4335it [00:00, 43346.42it/s] 8604it [00:00, 43273.96it/s] 8472it [00:00, 42497.95it/s] 8544it [00:00, 42901.65it/s] 8819it [00:00, 44219.97it/s] 12993it [00:00, 43553.41it/s] 12901it [00:00, 43312.81it/s] 12868it [00:00, 43050.63it/s] 13241it [00:00, 44092.33it/s] 17319it [00:00, 43099.45it/s] + 17233it [00:00, 43009.32it/s] 17174it [00:00, 42706.87it/s]2026-02-06 15:54:59.130 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1095 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ja_pack_new.jsonl + 17319it [00:00, 42835.46it/s] +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_ko with +2026-02-06 15:54:59.132 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1095 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ja_pack_new.jsonl + 0it [00:00, ?it/s] 17319it [00:00, 42648.83it/s] +2026-02-06 15:54:59.134 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1095 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ja_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_ko with + 0it [00:00, ?it/s]{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_ko with + 0it [00:00, ?it/s] 235it [00:00, 41105.19it/s] +2026-02-06 15:54:59.139 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 235 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ko_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_pt with + 0it [00:00, ?it/s] 235it [00:00, 56971.36it/s] +2026-02-06 15:54:59.140 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 235 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ko_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_pt with + 0it [00:00, ?it/s] 235it [00:00, 56172.65it/s] +2026-02-06 15:54:59.142 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 235 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ko_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_pt with + 0it [00:00, ?it/s] 17319it [00:00, 43751.77it/s] +2026-02-06 15:54:59.158 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1095 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ja_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_ko with + 0it [00:00, ?it/s] 235it [00:00, 55517.71it/s] +2026-02-06 15:54:59.165 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 235 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_ko_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset mls_pt with + 0it [00:00, ?it/s] 5309it [00:00, 124458.88it/s] 5309it [00:00, 126229.76it/s] + + 5309it [00:00, 132515.81it/s] +2026-02-06 15:54:59.183 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1061 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_pt_pack_new.jsonl +2026-02-06 15:54:59.183 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1061 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_pt_pack_new.jsonl +2026-02-06 15:54:59.183 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1061 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_pt_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11}loading dataset msr86k_ru with + +loading dataset msr86k_ru with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset msr86k_ru with + 0it [00:00, ?it/s] 0it [00:00, ?it/s] 0it [00:00, ?it/s] 5309it [00:00, 150808.03it/s] +2026-02-06 15:54:59.202 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 1061 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/mls_pt_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset msr86k_ru with + 0it [00:00, ?it/s] 4047it [00:00, 40458.37it/s] 4043it [00:00, 40421.17it/s] 4054it [00:00, 40538.40it/s] 4255it [00:00, 42545.68it/s] 8340it [00:00, 41909.77it/s] 8343it [00:00, 41934.19it/s] 8366it [00:00, 42054.21it/s] 8611it [00:00, 43140.52it/s] 12661it [00:00, 42501.36it/s] 12664it [00:00, 42515.70it/s] 12661it [00:00, 42457.66it/s] 12978it [00:00, 43380.60it/s] 16912it [00:00, 42200.94it/s] 16916it [00:00, 42199.20it/s] 16907it [00:00, 42068.69it/s] 17317it [00:00, 42681.16it/s] 21133it [00:00, 40392.34it/s] 21137it [00:00, 40391.66it/s] 21115it [00:00, 40131.87it/s] 21587it [00:00, 40636.82it/s] 25189it [00:00, 39379.52it/s] 25185it [00:00, 39354.35it/s] 25143it [00:00, 38990.23it/s] 25667it [00:00, 39556.95it/s] 29131it [00:00, 38713.27it/s] 29138it [00:00, 38706.20it/s] 29054it [00:00, 38311.96it/s] 29635it [00:00, 38826.83it/s] 33009it [00:00, 38237.78it/s] 33016it [00:00, 38235.00it/s] 32893it [00:00, 37829.53it/s] 33526it [00:00, 38238.40it/s] 36844it [00:00, 37945.21it/s] 36837it [00:00, 37888.97it/s] 36681it [00:00, 37571.01it/s] 37355it [00:00, 37931.29it/s] 40629it [00:01, 37466.94it/s] 40641it [00:01, 37447.66it/s] 40441it [00:01, 37197.19it/s] 41151it [00:01, 37827.14it/s] 44440it [00:01, 37655.19it/s] 44451it [00:01, 37640.56it/s] 44216it [00:01, 37360.65it/s] 44936it [00:01, 37815.64it/s] 48208it [00:01, 37633.57it/s] 48217it [00:01, 37627.31it/s] 47954it [00:01, 37314.05it/s] 48719it [00:01, 37746.34it/s] 51973it [00:01, 37543.72it/s] 51981it [00:01, 37537.55it/s] 51692it [00:01, 37332.86it/s] 52495it [00:01, 37672.12it/s] 55731it [00:01, 37552.76it/s] 55738it [00:01, 37544.83it/s] 55438it [00:01, 37367.23it/s] 56263it [00:01, 37559.83it/s] 59487it [00:01, 37411.37it/s] 59493it [00:01, 37406.31it/s] 59176it [00:01, 37258.77it/s] 60020it [00:01, 37484.41it/s] 63240it [00:01, 37445.17it/s] 63255it [00:01, 37468.56it/s] 62952it [00:01, 37406.02it/s] 63773it [00:01, 37497.20it/s] 67000it [00:01, 37488.67it/s] 67007it [00:01, 37482.02it/s] 66696it [00:01, 37415.40it/s] 67523it [00:01, 37482.39it/s] 70765it [00:01, 37534.14it/s] 70776it [00:01, 37542.49it/s] 70464it [00:01, 37492.61it/s] 71287it [00:01, 37528.16it/s] 74519it [00:01, 37518.68it/s] 74531it [00:01, 37438.51it/s] 74221it [00:01, 37513.37it/s] 75056it [00:01, 37573.43it/s] 78297it [00:02, 37594.30it/s] 78284it [00:02, 37462.57it/s] 77997it [00:02, 37586.45it/s] 78814it [00:02, 37556.93it/s] 82058it [00:02, 37542.90it/s] 82057it [00:02, 37479.10it/s] 81756it [00:02, 37462.41it/s] 82570it [00:02, 37532.71it/s] 85473it [00:02, 38265.16it/s] + 85473it [00:02, 38216.61it/s] + 85473it [00:02, 38032.60it/s] 85473it [00:02, 38356.47it/s] + +2026-02-06 15:55:01.446 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 35898 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/msr86k_ru_pack_new.jsonl +2026-02-06 15:55:01.450 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 35898 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/msr86k_ru_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_th with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_th with +2026-02-06 15:55:01.461 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 35898 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/msr86k_ru_pack_new.jsonl +2026-02-06 15:55:01.461 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 35898 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/msr86k_ru_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_th with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset com_voice_th with + 0it [00:00, ?it/s] 0it [00:00, ?it/s] 0it [00:00, ?it/s] 0it [00:00, ?it/s] 2900it [00:00, 28989.04it/s] 2990it [00:00, 29889.62it/s] 3014it [00:00, 30133.35it/s] 3028it [00:00, 30274.18it/s] 6246it [00:00, 31613.97it/s] 6328it [00:00, 31936.73it/s] 6332it [00:00, 31923.29it/s] 6338it [00:00, 31934.97it/s] 9755it [00:00, 33199.36it/s] 9882it [00:00, 33578.17it/s] 9803it [00:00, 33193.54it/s] 9793it [00:00, 33127.72it/s] 13535it [00:00, 35012.33it/s] 13651it [00:00, 35199.19it/s] 13570it [00:00, 34956.47it/s] 13589it [00:00, 35032.77it/s] 17398it [00:00, 35180.00it/s] 17398it [00:00, 34957.29it/s] + +2026-02-06 15:55:01.985 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 34796 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_th_pack_new.jsonl +2026-02-06 15:55:01.985 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 34796 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_th_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset msr86k_vi with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset msr86k_vi with + 0it [00:00, ?it/s] 0it [00:00, ?it/s] 17398it [00:00, 35064.47it/s] + 17398it [00:00, 35036.38it/s] +2026-02-06 15:55:01.998 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 34796 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_th_pack_new.jsonl +2026-02-06 15:55:01.998 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 34796 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/com_voice_th_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset msr86k_vi with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset msr86k_vi with + 0it [00:00, ?it/s] 0it [00:00, ?it/s] 3298it [00:00, 32972.01it/s] 3312it [00:00, 33117.50it/s] 3479it [00:00, 34781.16it/s] 3461it [00:00, 34598.98it/s] 6767it [00:00, 33980.18it/s] 6768it [00:00, 33963.02it/s] 6921it [00:00, 34256.85it/s] 6958it [00:00, 34348.30it/s] 10166it [00:00, 33895.13it/s] 10165it [00:00, 33767.50it/s] 10347it [00:00, 34098.25it/s] 10394it [00:00, 34006.12it/s] 13670it [00:00, 34344.15it/s] 13684it [00:00, 34323.35it/s] 13815it [00:00, 34322.25it/s] 13881it [00:00, 34342.15it/s] 17182it [00:00, 34623.01it/s] 17184it [00:00, 34565.96it/s] 17318it [00:00, 34574.27it/s] 17378it [00:00, 34563.07it/s] 20721it [00:00, 34881.74it/s] 20751it [00:00, 34937.36it/s] 20842it [00:00, 34799.03it/s] 20909it [00:00, 34813.15it/s] 24292it [00:00, 35150.93it/s] 24325it [00:00, 35196.11it/s] 24406it [00:00, 35071.42it/s] 24485it [00:00, 35120.42it/s] 27808it [00:00, 34414.78it/s] 27845it [00:00, 34408.11it/s] 27914it [00:00, 34232.72it/s] 27998it [00:00, 34280.42it/s] 31253it [00:00, 33472.72it/s] 31290it [00:00, 33496.45it/s] 31342it [00:00, 33374.58it/s] 31431it [00:00, 33353.66it/s] 34607it [00:01, 32959.69it/s] 34646it [00:01, 32935.70it/s] 34686it [00:01, 32777.06it/s] 34774it [00:01, 32770.14it/s] 37908it [00:01, 32538.04it/s] 37945it [00:01, 32563.70it/s] 37969it [00:01, 32462.11it/s] 38057it [00:01, 32407.32it/s] 41166it [00:01, 32256.01it/s] 41205it [00:01, 32281.31it/s] 41219it [00:01, 32173.51it/s] 41302it [00:01, 32190.29it/s] 44394it [00:01, 32137.09it/s] 44436it [00:01, 32137.27it/s] 44439it [00:01, 31707.97it/s] 44523it [00:01, 31724.76it/s] 47609it [00:01, 32009.43it/s] 47651it [00:01, 32009.57it/s] 47612it [00:01, 31706.95it/s] 47698it [00:01, 31664.14it/s] 50811it [00:01, 31932.71it/s] 50853it [00:01, 31925.96it/s] 50784it [00:01, 31698.05it/s] 50866it [00:01, 31653.17it/s] 54005it [00:01, 31824.26it/s] 54046it [00:01, 31826.70it/s] 53955it [00:01, 31620.46it/s] 54032it [00:01, 31596.93it/s] 57202it [00:01, 31862.50it/s] 57236it [00:01, 31845.94it/s] 57136it [00:01, 31674.87it/s] 57207it [00:01, 31640.53it/s] 60389it [00:01, 31645.54it/s] 60421it [00:01, 31643.73it/s] 60304it [00:01, 31551.87it/s] 60372it [00:01, 31544.38it/s] 63586it [00:01, 31622.87it/s] 63554it [00:01, 31407.57it/s] 63460it [00:01, 31502.79it/s] 63527it [00:01, 31511.31it/s] 66780it [00:02, 31714.10it/s] 66744it [00:02, 31549.67it/s] 66611it [00:02, 31448.19it/s] 66701it [00:02, 31575.82it/s] 69952it [00:02, 31622.87it/s] 69900it [00:02, 31518.00it/s] 69807it [00:02, 31599.36it/s] 69862it [00:02, 31584.02it/s] 73115it [00:02, 31594.97it/s] 73081it [00:02, 31603.91it/s] 72968it [00:02, 31546.12it/s] 73021it [00:02, 31550.76it/s] 76290it [00:02, 31637.85it/s] 76301it [00:02, 31781.42it/s] 76151it [00:02, 31627.90it/s] 76211it [00:02, 31654.90it/s] 79454it [00:02, 31625.15it/s] 79480it [00:02, 31723.95it/s] 79314it [00:02, 31626.35it/s] 79377it [00:02, 31587.98it/s] 82667it [00:02, 31773.45it/s] 82692it [00:02, 31840.36it/s] 82514it [00:02, 31735.75it/s] 82573it [00:02, 31697.17it/s] 85871it [00:02, 31851.34it/s] 85898it [00:02, 31904.40it/s] 85688it [00:02, 31687.85it/s] 85746it [00:02, 31704.47it/s] 89057it [00:02, 31780.49it/s] 89089it [00:02, 31802.06it/s] 88863it [00:02, 31703.00it/s] 88917it [00:02, 31656.35it/s] 92253it [00:02, 31833.55it/s] 92277it [00:02, 31821.72it/s] 92034it [00:02, 31635.75it/s] 92089it [00:02, 31673.74it/s] 95457it [00:02, 31893.77it/s] 95496it [00:02, 31928.44it/s] 95219it [00:02, 31697.42it/s] 95270it [00:02, 31711.08it/s] 98657it [00:03, 31924.62it/s] 98694it [00:03, 31942.49it/s] 98417it [00:03, 31779.75it/s] 98456it [00:03, 31752.69it/s] 101850it [00:03, 31849.43it/s] 101889it [00:03, 31863.02it/s] 101596it [00:03, 31691.11it/s] 101632it [00:03, 31713.65it/s] 105036it [00:03, 31838.45it/s] 105076it [00:03, 31852.35it/s] 104780it [00:03, 31732.35it/s] 104805it [00:03, 31718.08it/s] 108227it [00:03, 31858.88it/s] 108272it [00:03, 31880.94it/s] 107960it [00:03, 31749.26it/s] 107980it [00:03, 31724.73it/s] 111431it [00:03, 31911.74it/s] 111472it [00:03, 31915.45it/s] 111136it [00:03, 31750.78it/s] 111173it [00:03, 31785.63it/s] 114628it [00:03, 31927.63it/s] 114671it [00:03, 31936.06it/s] 114326it [00:03, 31794.53it/s] 114352it [00:03, 31776.41it/s] 117857it [00:03, 32033.39it/s] 117897it [00:03, 32031.96it/s] 117542it [00:03, 31902.36it/s] 117565it [00:03, 31881.56it/s] 121061it [00:03, 32027.69it/s] 121101it [00:03, 32010.49it/s] 120737it [00:03, 31914.64it/s] 120754it [00:03, 31880.34it/s] 124264it [00:03, 31998.65it/s] 124303it [00:03, 31993.39it/s] 123929it [00:03, 31907.59it/s] 123943it [00:03, 31879.81it/s] 127468it [00:03, 32008.45it/s] 127510it [00:03, 32015.37it/s] 127120it [00:03, 31871.11it/s] 127131it [00:03, 31859.17it/s] 130669it [00:04, 31919.20it/s] 130712it [00:04, 31930.28it/s] 130308it [00:04, 31763.59it/s] 130317it [00:04, 31753.41it/s] 133861it [00:04, 31648.72it/s] 133906it [00:04, 31493.00it/s] 133485it [00:04, 31604.98it/s] 133493it [00:04, 31612.31it/s] 137027it [00:04, 31563.31it/s] 137061it [00:04, 31508.98it/s] 136646it [00:04, 31564.94it/s] 136655it [00:04, 31552.02it/s] 140239it [00:04, 31728.46it/s] 140295it [00:04, 31754.36it/s] 139832it [00:04, 31649.85it/s] 139841it [00:04, 31642.00it/s] 141611it [00:04, 32257.06it/s] + 141611it [00:04, 32257.66it/s] + 141611it [00:04, 32155.62it/s] 141611it [00:04, 32157.10it/s] + +2026-02-06 15:55:06.409 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 36252 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/msr86k_vi_pack_new.jsonl +2026-02-06 15:55:06.409 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 36252 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/msr86k_vi_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset librispeech_en with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset librispeech_en with +2026-02-06 15:55:06.436 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 36252 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/msr86k_vi_pack_new.jsonl +2026-02-06 15:55:06.436 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 36252 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/msr86k_vi_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset librispeech_en with +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +loading dataset librispeech_en with + 0it [00:00, ?it/s] 0it [00:00, ?it/s] 0it [00:00, ?it/s] 0it [00:00, ?it/s] 7360it [00:00, 73592.70it/s] 7610it [00:00, 76096.26it/s] 7162it [00:00, 71613.41it/s] 7309it [00:00, 73086.76it/s] 21963it [00:00, 116196.97it/s] 22258it [00:00, 117490.11it/s] 21171it [00:00, 111885.08it/s] 21283it [00:00, 112291.38it/s] 30983it [00:00, 121644.30it/s] 30983it [00:00, 120784.35it/s] + +2026-02-06 15:55:06.749 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 30983 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/librispeech_en_pack_new.jsonl +2026-02-06 15:55:06.749 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 30983 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/librispeech_en_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11}{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} + + 30983it [00:00, 119658.72it/s] 30983it [00:00, 119319.45it/s] + +2026-02-06 15:55:06.780 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 30983 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/librispeech_en_pack_new.jsonl +2026-02-06 15:55:06.780 | INFO | data.qwen_audio_pretrain_dataset_pack:load_meta_dicts:84 - loaded dataset of 30983 samples from /fs/nlp/common_intern/meiyuxiang/data_prepare/step3_pack_2048_scale_4_fixed_1212/librispeech_en_pack_new.jsonl +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +{'en': 0, 'fr': 1, 'ru': 2, 'th': 3, 'de': 4, 'es': 5, 'it': 6, 'pt': 7, 'ar': 8, 'vi': 9, 'ja': 10, 'ko': 11} +AudioEncoderConfig(add_audio_bos_eos_token=True, audio_attn_implementation='flash_attention_2', audio_start_id=155163, avg_pool=True, is_causal=False, n_ctx=1500, n_head=20, n_layer=32, n_mels=128, n_state=1280, output_dim=2048, post_subsampling_scale=4, use_conformer_encoder=False, use_wav2vec2_encoder=False, wav2vec2_model_dim=None, wav2vec2_num_heads=None, wav2vec2_num_layers=None, wav2vec2_ffn_dim=None, use_conv_subsampling=True, use_chunk_encoder=False, chunk_n_window=100, conv_chunksize=200, chunk_n_window_candidates=[50, 100, 200, 400], randomize_chunk_window=False, chunk_level_router=False, num_moe_experts=8) +AudioEncoderConfig(add_audio_bos_eos_token=True, audio_attn_implementation='flash_attention_2', audio_start_id=155163, avg_pool=True, is_causal=False, n_ctx=1500, n_head=20, n_layer=32, n_mels=128, n_state=1280, output_dim=2048, post_subsampling_scale=4, use_conformer_encoder=False, use_wav2vec2_encoder=False, wav2vec2_model_dim=None, wav2vec2_num_heads=None, wav2vec2_num_layers=None, wav2vec2_ffn_dim=None, use_conv_subsampling=True, use_chunk_encoder=False, chunk_n_window=100, conv_chunksize=200, chunk_n_window_candidates=[50, 100, 200, 400], randomize_chunk_window=False, chunk_level_router=False, num_moe_experts=8) +2026-02-06 15:55:07.043 | INFO | model.unigpt_audio_models.audio:__init__:639 - initializing AudioEncoder with audio_attn_implementation=flash_attention_2, is_causal=False +AudioEncoderConfig(add_audio_bos_eos_token=True, audio_attn_implementation='flash_attention_2', audio_start_id=155163, avg_pool=True, is_causal=False, n_ctx=1500, n_head=20, n_layer=32, n_mels=128, n_state=1280, output_dim=2048, post_subsampling_scale=4, use_conformer_encoder=False, use_wav2vec2_encoder=False, wav2vec2_model_dim=None, wav2vec2_num_heads=None, wav2vec2_num_layers=None, wav2vec2_ffn_dim=None, use_conv_subsampling=True, use_chunk_encoder=False, chunk_n_window=100, conv_chunksize=200, chunk_n_window_candidates=[50, 100, 200, 400], randomize_chunk_window=False, chunk_level_router=False, num_moe_experts=8) +2026-02-06 15:55:07.051 | INFO | model.unigpt_audio_models.audio:__init__:639 - initializing AudioEncoder with audio_attn_implementation=flash_attention_2, is_causal=False +2026-02-06 15:55:07.059 | INFO | model.unigpt_audio_models.audio:__init__:639 - initializing AudioEncoder with audio_attn_implementation=flash_attention_2, is_causal=False +AudioEncoderConfig(add_audio_bos_eos_token=True, audio_attn_implementation='flash_attention_2', audio_start_id=155163, avg_pool=True, is_causal=False, n_ctx=1500, n_head=20, n_layer=32, n_mels=128, n_state=1280, output_dim=2048, post_subsampling_scale=4, use_conformer_encoder=False, use_wav2vec2_encoder=False, wav2vec2_model_dim=None, wav2vec2_num_heads=None, wav2vec2_num_layers=None, wav2vec2_ffn_dim=None, use_conv_subsampling=True, use_chunk_encoder=False, chunk_n_window=100, conv_chunksize=200, chunk_n_window_candidates=[50, 100, 200, 400], randomize_chunk_window=False, chunk_level_router=False, num_moe_experts=8) +2026-02-06 15:55:07.100 | INFO | model.unigpt_audio_models.audio:__init__:639 - initializing AudioEncoder with audio_attn_implementation=flash_attention_2, is_causal=False + Loading checkpoint shards: 0%| | 0/2 [00:00 +[2026-02-06 15:55:42,030] [INFO] [logging.py:96:log_dist] [Rank 0] Creating torch.bfloat16 ZeRO stage 2 optimizer +[2026-02-06 15:55:42,030] [INFO] [stage_1_and_2.py:149:__init__] Reduce bucket size 500,000,000 +[2026-02-06 15:55:42,030] [INFO] [stage_1_and_2.py:150:__init__] Allgather bucket size 500,000,000 +[2026-02-06 15:55:42,030] [INFO] [stage_1_and_2.py:151:__init__] CPU Offload: False +[2026-02-06 15:55:42,030] [INFO] [stage_1_and_2.py:152:__init__] Round robin gradient partitioning: False +[2026-02-06 15:55:59,616] [WARNING] [engine.py:2740:load_checkpoint] Unable to find latest file at /fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +2026-02-06 15:56:00.218 | INFO | trainer.unigpt_pretrain_trainer:fit:130 - [Zipper LoRA] Freeze B weights for 0 steps. +[2026-02-06 15:56:03,224] [WARNING] [engine.py:2740:load_checkpoint] Unable to find latest file at /fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[2026-02-06 15:56:03,278] [WARNING] [engine.py:2740:load_checkpoint] Unable to find latest file at /fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +[2026-02-06 15:56:03,417] [INFO] [utils.py:800:see_memory_usage] Before initializing optimizer states +[2026-02-06 15:56:03,418] [INFO] [utils.py:801:see_memory_usage] MA 5.54 GB Max_MA 5.54 GB CA 6.16 GB Max_CA 6 GB +[2026-02-06 15:56:03,419] [INFO] [utils.py:808:see_memory_usage] CPU Virtual Memory: used = 0.0 GB, percent = 0.0% +2026-02-06 15:56:03.423 | INFO | trainer.unigpt_pretrain_trainer:fit:130 - [Zipper LoRA] Freeze B weights for 0 steps. +2026-02-06 15:56:03.477 | INFO | trainer.unigpt_pretrain_trainer:fit:130 - [Zipper LoRA] Freeze B weights for 0 steps. +[2026-02-06 15:56:03,595] [INFO] [utils.py:800:see_memory_usage] After initializing optimizer states +[2026-02-06 15:56:03,596] [INFO] [utils.py:801:see_memory_usage] MA 5.54 GB Max_MA 5.92 GB CA 6.53 GB Max_CA 7 GB +[2026-02-06 15:56:03,597] [INFO] [utils.py:808:see_memory_usage] CPU Virtual Memory: used = 0.0 GB, percent = 0.0% +[2026-02-06 15:56:03,597] [INFO] [stage_1_and_2.py:539:__init__] optimizer state initialized +[2026-02-06 15:56:03,763] [INFO] [utils.py:800:see_memory_usage] After initializing ZeRO optimizer +[2026-02-06 15:56:03,764] [INFO] [utils.py:801:see_memory_usage] MA 5.54 GB Max_MA 5.54 GB CA 6.53 GB Max_CA 7 GB +[2026-02-06 15:56:03,764] [INFO] [utils.py:808:see_memory_usage] CPU Virtual Memory: used = 0.0 GB, percent = 0.0% +[2026-02-06 15:56:03,777] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed Final Optimizer = FusedAdam +[2026-02-06 15:56:03,777] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed using client LR scheduler +[2026-02-06 15:56:03,777] [INFO] [logging.py:96:log_dist] [Rank 0] DeepSpeed LR Scheduler = +[2026-02-06 15:56:03,777] [INFO] [logging.py:96:log_dist] [Rank 0] step=0, skipped=0, lr=[0.0, 0.0], mom=[(0.9, 0.95), (0.9, 0.95)] +[2026-02-06 15:56:03,783] [INFO] [config.py:987:print] DeepSpeedEngine configuration: +[2026-02-06 15:56:03,783] [INFO] [config.py:991:print] activation_checkpointing_config { + "partition_activations": false, + "contiguous_memory_optimization": false, + "cpu_checkpointing": false, + "number_checkpoints": null, + "synchronize_checkpoint_boundary": false, + "profile": false +} +[2026-02-06 15:56:03,783] [INFO] [config.py:991:print] aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True} +[2026-02-06 15:56:03,783] [INFO] [config.py:991:print] amp_enabled .................. False +[2026-02-06 15:56:03,783] [INFO] [config.py:991:print] amp_params ................... False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] autotuning_config ............ { + "enabled": false, + "start_step": null, + "end_step": null, + "metric_path": null, + "arg_mappings": null, + "metric": "throughput", + "model_info": null, + "results_dir": "autotuning_results", + "exps_dir": "autotuning_exps", + "overwrite": true, + "fast": true, + "start_profile_step": 3, + "end_profile_step": 5, + "tuner_type": "gridsearch", + "tuner_early_stopping": 5, + "tuner_num_trials": 50, + "model_info_path": null, + "mp_size": 1, + "max_train_batch_size": null, + "min_train_batch_size": 1, + "max_train_micro_batch_size_per_gpu": 1.024000e+03, + "min_train_micro_batch_size_per_gpu": 1, + "num_tuning_micro_batch_sizes": 3 +} +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] bfloat16_enabled ............. True +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] checkpoint_parallel_write_pipeline False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] checkpoint_tag_validation_enabled True +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] checkpoint_tag_validation_fail False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] comms_config ................. +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] communication_data_type ...... None +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] compile_config ............... enabled=False backend='inductor' kwargs={} +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] compression_config ........... {'weight_quantization': {'shared_parameters': {'enabled': False, 'quantizer_kernel': False, 'schedule_offset': 0, 'quantize_groups': 1, 'quantize_verbose': False, 'quantization_type': 'symmetric', 'quantize_weight_in_forward': False, 'rounding': 'nearest', 'fp16_mixed_quantize': False, 'quantize_change_ratio': 0.001}, 'different_groups': {}}, 'activation_quantization': {'shared_parameters': {'enabled': False, 'quantization_type': 'symmetric', 'range_calibration': 'dynamic', 'schedule_offset': 1000}, 'different_groups': {}}, 'sparse_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'row_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'head_pruning': {'shared_parameters': {'enabled': False, 'method': 'topk', 'schedule_offset': 1000}, 'different_groups': {}}, 'channel_pruning': {'shared_parameters': {'enabled': False, 'method': 'l1', 'schedule_offset': 1000}, 'different_groups': {}}, 'layer_reduction': {'enabled': False}} +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] curriculum_enabled_legacy .... False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] curriculum_params_legacy ..... False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] data_efficiency_config ....... {'enabled': False, 'seed': 1234, 'data_sampling': {'enabled': False, 'num_epochs': 1000, 'num_workers': 0, 'curriculum_learning': {'enabled': False}}, 'data_routing': {'enabled': False, 'random_ltd': {'enabled': False, 'layer_token_lr_schedule': {'enabled': False}}}} +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] data_efficiency_enabled ...... False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] dataloader_drop_last ......... False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] disable_allgather ............ False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] dump_state ................... False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] dynamic_loss_scale_args ...... None +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] eigenvalue_enabled ........... False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] eigenvalue_gas_boundary_resolution 1 +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] eigenvalue_layer_name ........ bert.encoder.layer +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] eigenvalue_layer_num ......... 0 +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] eigenvalue_max_iter .......... 100 +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] eigenvalue_stability ......... 1e-06 +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] eigenvalue_tol ............... 0.01 +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] eigenvalue_verbose ........... False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] elasticity_enabled ........... False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] flops_profiler_config ........ { + "enabled": false, + "recompute_fwd_factor": 0.0, + "profile_step": 1, + "module_depth": -1, + "top_modules": 1, + "detailed": true, + "output_file": null +} +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] fp16_auto_cast ............... None +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] fp16_enabled ................. False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] fp16_master_weights_and_gradients False +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] global_rank .................. 0 +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] grad_accum_dtype ............. fp32 +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] gradient_accumulation_steps .. 2 +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] gradient_clipping ............ 1.0 +[2026-02-06 15:56:03,784] [INFO] [config.py:991:print] gradient_predivide_factor .... 1.0 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] graph_harvesting ............. False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] hybrid_engine ................ enabled=False max_out_tokens=512 inference_tp_size=1 release_inference_cache=False pin_parameters=True tp_gather_partition_size=8 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] initial_dynamic_scale ........ 1 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] load_universal_checkpoint .... False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] loss_scale ................... 1.0 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] memory_breakdown ............. False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] mics_hierarchial_params_gather False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] mics_shard_size .............. -1 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] monitor_config ............... tensorboard=TensorBoardConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') wandb=WandbConfig(enabled=False, group=None, team=None, project='deepspeed') csv_monitor=CSVConfig(enabled=False, output_path='', job_name='DeepSpeedJobName') enabled=False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] nebula_config ................ { + "enabled": false, + "persistent_storage_path": null, + "persistent_time_interval": 100, + "num_of_version_in_retention": 2, + "enable_nebula_load": true, + "load_path": null +} +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] optimizer_legacy_fusion ...... False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] optimizer_name ............... None +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] optimizer_params ............. None +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0, 'pipe_partitioned': True, 'grad_partitioned': True} +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] pld_enabled .................. False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] pld_params ................... False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] prescale_gradients ........... False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] scheduler_name ............... None +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] scheduler_params ............. None +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] seq_parallel_communication_data_type torch.float32 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] sparse_attention ............. None +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] sparse_gradients_enabled ..... False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] steps_per_print .............. 100 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] train_batch_size ............. 80 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] train_micro_batch_size_per_gpu 10 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] use_data_before_expert_parallel_ False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] use_node_local_storage ....... False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] wall_clock_breakdown ......... False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] weight_quantization_config ... None +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] world_size ................... 4 +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] zero_allow_untested_optimizer False +[2026-02-06 15:56:03,785] [INFO] [config.py:991:print] zero_config .................. stage=2 contiguous_gradients=True reduce_scatter=True reduce_bucket_size=500,000,000 use_multi_rank_bucket_allreduce=True allgather_partitions=True allgather_bucket_size=500,000,000 overlap_comm=False load_from_fp32_weights=True elastic_checkpoint=False offload_param=DeepSpeedZeroOffloadParamConfig(device='none', nvme_path=None, buffer_count=5, buffer_size=100,000,000, max_in_cpu=1,000,000,000, pin_memory=False) offload_optimizer=DeepSpeedZeroOffloadOptimizerConfig(device='none', nvme_path=None, buffer_count=4, pin_memory=True, pipeline=False, pipeline_read=False, pipeline_write=False, fast_init=False, ratio=1.0) sub_group_size=1,000,000,000 cpu_offload_param=None cpu_offload_use_pin_memory=None cpu_offload=None prefetch_bucket_size=50,000,000 param_persistence_threshold=100,000 model_persistence_threshold=sys.maxsize max_live_parameters=1,000,000,000 max_reuse_distance=1,000,000,000 gather_16bit_weights_on_model_save=False stage3_gather_fp16_weights_on_model_save=False ignore_unused_parameters=True legacy_stage1=False round_robin_gradients=False zero_hpz_partition_size=1 zero_quantized_weights=False zero_quantized_nontrainable_weights=False zero_quantized_gradients=False mics_shard_size=-1 mics_hierarchical_params_gather=False memory_efficient_linear=True pipeline_loading_checkpoint=False override_module_apply=True +[2026-02-06 15:56:03,786] [INFO] [config.py:991:print] zero_enabled ................. True +[2026-02-06 15:56:03,786] [INFO] [config.py:991:print] zero_force_ds_cpu_optimizer .. True +[2026-02-06 15:56:03,786] [INFO] [config.py:991:print] zero_optimization_stage ...... 2 +[2026-02-06 15:56:03,787] [INFO] [config.py:977:print_user_config] json = { + "steps_per_print": 100, + "zero_optimization": { + "stage": 2, + "offload_param": { + "device": "none" + }, + "offload_optimizer": { + "device": "none", + "pin_memory": true + }, + "sub_group_size": "auto", + "stage3_max_live_parameters": "auto", + "stage3_max_reuse_distance": "auto", + "stage3_param_persistence_threshold": "auto", + "stage3_prefetch_bucket_size": "auto", + "reduce_bucket_size": "auto", + "zero_hpz_partition_size": 1, + "zero_quantized_weights": false, + "zero_quantized_gradients": false + }, + "bf16": { + "enabled": true + }, + "gradient_clipping": 1.0, + "prescale_gradients": false, + "wall_clock_breakdown": false, + "data_types": { + "grad_accum_dtype": "fp32" + }, + "train_micro_batch_size_per_gpu": 10, + "train_batch_size": 80 +} +Load checkpoint: /fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked +[2026-02-06 15:56:03,787] [WARNING] [engine.py:2740:load_checkpoint] Unable to find latest file at /fs/nlp/common_intern/meiyuxiang/assets/multilingual/qwen3-1.7b-whisper-0205_12x1000h_lite30h_zipper_lora_independent_audio_init_baseline_with_lid_embedding_non_chunked/ckpts/latest, if trying to load latest checkpoint please ensure this file exists or pass an explicit checkpoint tag when loading a checkpoint. +> setting tensorboard ... +2026-02-06 15:56:03.969 | INFO | trainer.unigpt_pretrain_trainer:fit:130 - [Zipper LoRA] Freeze B weights for 0 steps. + Train epoch: 0%| | 0/3 [00:00