[2026-03-22 02:39:15,958] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) [2026-03-22 02:39:19,195] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False [2026-03-22 02:39:20,649] [INFO] [runner.py:499:main] Using IP address of 26.0.224.28 for node 10.48.48.92 [2026-03-22 02:39:20,650] [INFO] [multinode_runner.py:85:get_cmd] Running on the following workers: 10.48.48.92,10.48.54.166,10.48.53.25,10.48.53.162,10.48.53.153,10.48.52.101,10.48.48.37,10.48.48.32 [2026-03-22 02:39:20,650] [INFO] [runner.py:610:main] cmd = pdsh -S -f 1024 -w 10.48.48.92,10.48.54.166,10.48.53.25,10.48.53.162,10.48.53.153,10.48.52.101,10.48.48.37,10.48.48.32 export NCCL_VERSION=2.18.3-1; export NCCL_SOCKET_IFNAME=eth; export NCCL_IB_HCA=mlx5; export NCCL_IB_GID_INDEX=3; export NCCL_IB_DISABLE=0; export PYTHONPATH=/nlp_group/yueyang/code/swe/ms-swift-370; cd /nlp_group/yueyang/code/swe/ms-swift-370; /mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10 -u -m deepspeed.launcher.launch --world_info=eyIxMC40OC40OC45MiI6IFswLCAxLCAyLCAzLCA0LCA1LCA2LCA3XSwgIjEwLjQ4LjU0LjE2NiI6IFswLCAxLCAyLCAzLCA0LCA1LCA2LCA3XSwgIjEwLjQ4LjUzLjI1IjogWzAsIDEsIDIsIDMsIDQsIDUsIDYsIDddLCAiMTAuNDguNTMuMTYyIjogWzAsIDEsIDIsIDMsIDQsIDUsIDYsIDddLCAiMTAuNDguNTMuMTUzIjogWzAsIDEsIDIsIDMsIDQsIDUsIDYsIDddLCAiMTAuNDguNTIuMTAxIjogWzAsIDEsIDIsIDMsIDQsIDUsIDYsIDddLCAiMTAuNDguNDguMzciOiBbMCwgMSwgMiwgMywgNCwgNSwgNiwgN10sICIxMC40OC40OC4zMiI6IFswLCAxLCAyLCAzLCA0LCA1LCA2LCA3XX0= --node_rank=%n --master_addr=26.0.224.28 --master_port=29500 --enable_each_rank_log=None swift/cli/sft.py --model /nlp_group/yueyang/model/Qwen3-32B --model_type qwen3 --torch_dtype bfloat16 --train_type full --use_chat_template true --dataset /mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl --per_device_train_batch_size 1 --data_seed 42 --weight_decay 0.1 --learning_rate 1e-5 --attn_impl flash_attn --deepspeed zero3 --gradient_accumulation_steps 4 --dataloader_num_workers 8 --dataset_num_proc 16 --dataloader_pin_memory true --dataloader_persistent_workers true --dataloader_prefetch_factor 2 --warmup_ratio 0.001 --dataset_num_proc 64 --save_total_limit 10 --max_length 65536 --max_model_len 65536 --rope_scaling yarn --sequence_parallel_size 2 --truncation_strategy delete --split_dataset_ratio 0 --output_dir /mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch --eval_strategy no --use_liger_kernel true --use_hf --save_strategy steps --save_steps 100 --num_train_epochs 5 --ddp_timeout 180000 --ignore_data_skip true 10.48.48.37: [2026-03-22 02:39:23,806] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.54.166: [2026-03-22 02:39:24,032] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.162: [2026-03-22 02:39:24,105] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:24,134] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.92: [2026-03-22 02:39:25,033] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:26,784] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.52.101: [2026-03-22 02:39:27,025] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:27,377] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.25: [2026-03-22 02:39:27,461] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.162: [2026-03-22 02:39:27,536] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:39:27,754] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:27,905] [INFO] [launch.py:139:main] 6 NCCL_VERSION=2.18.3-1 10.48.48.37: [2026-03-22 02:39:27,905] [INFO] [launch.py:139:main] 6 NCCL_SOCKET_IFNAME=eth 10.48.48.37: [2026-03-22 02:39:27,905] [INFO] [launch.py:139:main] 6 NCCL_IB_HCA=mlx5 10.48.48.37: [2026-03-22 02:39:27,905] [INFO] [launch.py:139:main] 6 NCCL_IB_GID_INDEX=3 10.48.48.37: [2026-03-22 02:39:27,905] [INFO] [launch.py:139:main] 6 NCCL_IB_DISABLE=0 10.48.48.37: [2026-03-22 02:39:27,905] [INFO] [launch.py:146:main] WORLD INFO DICT: {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.25': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.162': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.153': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.52.101': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.37': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.32': [0, 1, 2, 3, 4, 5, 6, 7]} 10.48.48.37: [2026-03-22 02:39:27,905] [INFO] [launch.py:152:main] nnodes=8, num_local_procs=8, node_rank=6 10.48.48.37: [2026-03-22 02:39:27,906] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(, {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [8, 9, 10, 11, 12, 13, 14, 15], '10.48.53.25': [16, 17, 18, 19, 20, 21, 22, 23], '10.48.53.162': [24, 25, 26, 27, 28, 29, 30, 31], '10.48.53.153': [32, 33, 34, 35, 36, 37, 38, 39], '10.48.52.101': [40, 41, 42, 43, 44, 45, 46, 47], '10.48.48.37': [48, 49, 50, 51, 52, 53, 54, 55], '10.48.48.32': [56, 57, 58, 59, 60, 61, 62, 63]}) 10.48.48.37: [2026-03-22 02:39:27,906] [INFO] [launch.py:164:main] dist_world_size=64 10.48.48.37: [2026-03-22 02:39:27,906] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 10.48.48.37: [2026-03-22 02:39:27,906] [INFO] [launch.py:256:main] process 8916 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=0', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.37: [2026-03-22 02:39:27,907] [INFO] [launch.py:256:main] process 8917 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=1', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.37: [2026-03-22 02:39:27,907] [INFO] [launch.py:256:main] process 8918 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=2', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.37: [2026-03-22 02:39:27,907] [INFO] [launch.py:256:main] process 8919 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=3', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.37: [2026-03-22 02:39:27,908] [INFO] [launch.py:256:main] process 8920 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=4', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.37: [2026-03-22 02:39:27,908] [INFO] [launch.py:256:main] process 8921 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=5', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.37: [2026-03-22 02:39:27,909] [INFO] [launch.py:256:main] process 8922 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=6', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.37: [2026-03-22 02:39:27,909] [INFO] [launch.py:256:main] process 8923 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=7', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:139:main] 5 NCCL_VERSION=2.18.3-1 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:139:main] 5 NCCL_SOCKET_IFNAME=eth 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:139:main] 5 NCCL_IB_HCA=mlx5 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:139:main] 5 NCCL_IB_GID_INDEX=3 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:139:main] 5 NCCL_IB_DISABLE=0 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:146:main] WORLD INFO DICT: {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.25': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.162': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.153': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.52.101': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.37': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.32': [0, 1, 2, 3, 4, 5, 6, 7]} 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:152:main] nnodes=8, num_local_procs=8, node_rank=5 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(, {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [8, 9, 10, 11, 12, 13, 14, 15], '10.48.53.25': [16, 17, 18, 19, 20, 21, 22, 23], '10.48.53.162': [24, 25, 26, 27, 28, 29, 30, 31], '10.48.53.153': [32, 33, 34, 35, 36, 37, 38, 39], '10.48.52.101': [40, 41, 42, 43, 44, 45, 46, 47], '10.48.48.37': [48, 49, 50, 51, 52, 53, 54, 55], '10.48.48.32': [56, 57, 58, 59, 60, 61, 62, 63]}) 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:164:main] dist_world_size=64 10.48.52.101: [2026-03-22 02:39:28,137] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 10.48.52.101: [2026-03-22 02:39:28,138] [INFO] [launch.py:256:main] process 8887 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=0', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.52.101: [2026-03-22 02:39:28,138] [INFO] [launch.py:256:main] process 8888 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=1', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.52.101: [2026-03-22 02:39:28,139] [INFO] [launch.py:256:main] process 8889 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=2', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.52.101: [2026-03-22 02:39:28,139] [INFO] [launch.py:256:main] process 8890 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=3', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.52.101: [2026-03-22 02:39:28,140] [INFO] [launch.py:256:main] process 8891 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=4', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.52.101: [2026-03-22 02:39:28,140] [INFO] [launch.py:256:main] process 8892 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=5', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.52.101: [2026-03-22 02:39:28,141] [INFO] [launch.py:256:main] process 8893 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=6', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.52.101: [2026-03-22 02:39:28,141] [INFO] [launch.py:256:main] process 8894 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=7', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.92: [2026-03-22 02:39:28,362] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:139:main] 1 NCCL_VERSION=2.18.3-1 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:139:main] 1 NCCL_SOCKET_IFNAME=eth 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:139:main] 1 NCCL_IB_HCA=mlx5 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:139:main] 1 NCCL_IB_GID_INDEX=3 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:139:main] 1 NCCL_IB_DISABLE=0 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:146:main] WORLD INFO DICT: {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.25': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.162': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.153': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.52.101': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.37': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.32': [0, 1, 2, 3, 4, 5, 6, 7]} 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:152:main] nnodes=8, num_local_procs=8, node_rank=1 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(, {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [8, 9, 10, 11, 12, 13, 14, 15], '10.48.53.25': [16, 17, 18, 19, 20, 21, 22, 23], '10.48.53.162': [24, 25, 26, 27, 28, 29, 30, 31], '10.48.53.153': [32, 33, 34, 35, 36, 37, 38, 39], '10.48.52.101': [40, 41, 42, 43, 44, 45, 46, 47], '10.48.48.37': [48, 49, 50, 51, 52, 53, 54, 55], '10.48.48.32': [56, 57, 58, 59, 60, 61, 62, 63]}) 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:164:main] dist_world_size=64 10.48.54.166: [2026-03-22 02:39:28,703] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 10.48.54.166: [2026-03-22 02:39:28,704] [INFO] [launch.py:256:main] process 8950 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=0', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.54.166: [2026-03-22 02:39:28,704] [INFO] [launch.py:256:main] process 8951 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=1', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.54.166: [2026-03-22 02:39:28,705] [INFO] [launch.py:256:main] process 8952 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=2', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.54.166: [2026-03-22 02:39:28,705] [INFO] [launch.py:256:main] process 8953 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=3', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.54.166: [2026-03-22 02:39:28,706] [INFO] [launch.py:256:main] process 8954 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=4', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.54.166: [2026-03-22 02:39:28,706] [INFO] [launch.py:256:main] process 8955 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=5', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.54.166: [2026-03-22 02:39:28,706] [INFO] [launch.py:256:main] process 8956 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=6', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.54.166: [2026-03-22 02:39:28,707] [INFO] [launch.py:256:main] process 8957 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=7', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.153: [2026-03-22 02:39:28,707] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:139:main] 3 NCCL_VERSION=2.18.3-1 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:139:main] 3 NCCL_SOCKET_IFNAME=eth 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:139:main] 3 NCCL_IB_HCA=mlx5 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:139:main] 3 NCCL_IB_GID_INDEX=3 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:139:main] 3 NCCL_IB_DISABLE=0 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:146:main] WORLD INFO DICT: {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.25': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.162': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.153': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.52.101': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.37': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.32': [0, 1, 2, 3, 4, 5, 6, 7]} 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:152:main] nnodes=8, num_local_procs=8, node_rank=3 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(, {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [8, 9, 10, 11, 12, 13, 14, 15], '10.48.53.25': [16, 17, 18, 19, 20, 21, 22, 23], '10.48.53.162': [24, 25, 26, 27, 28, 29, 30, 31], '10.48.53.153': [32, 33, 34, 35, 36, 37, 38, 39], '10.48.52.101': [40, 41, 42, 43, 44, 45, 46, 47], '10.48.48.37': [48, 49, 50, 51, 52, 53, 54, 55], '10.48.48.32': [56, 57, 58, 59, 60, 61, 62, 63]}) 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:164:main] dist_world_size=64 10.48.53.162: [2026-03-22 02:39:28,931] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 10.48.53.162: [2026-03-22 02:39:28,932] [INFO] [launch.py:256:main] process 8888 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=0', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.162: [2026-03-22 02:39:28,932] [INFO] [launch.py:256:main] process 8889 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=1', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.162: [2026-03-22 02:39:28,932] [INFO] [launch.py:256:main] process 8890 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=2', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.162: [2026-03-22 02:39:28,933] [INFO] [launch.py:256:main] process 8891 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=3', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.162: [2026-03-22 02:39:28,933] [INFO] [launch.py:256:main] process 8892 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=4', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.162: [2026-03-22 02:39:28,933] [INFO] [launch.py:256:main] process 8893 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=5', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.162: [2026-03-22 02:39:28,934] [INFO] [launch.py:256:main] process 8894 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=6', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.162: [2026-03-22 02:39:28,934] [INFO] [launch.py:256:main] process 8895 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=7', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:139:main] 0 NCCL_VERSION=2.18.3-1 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:139:main] 0 NCCL_SOCKET_IFNAME=eth 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:139:main] 0 NCCL_IB_HCA=mlx5 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:139:main] 0 NCCL_IB_GID_INDEX=3 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:139:main] 0 NCCL_IB_DISABLE=0 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:146:main] WORLD INFO DICT: {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.25': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.162': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.153': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.52.101': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.37': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.32': [0, 1, 2, 3, 4, 5, 6, 7]} 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:152:main] nnodes=8, num_local_procs=8, node_rank=0 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(, {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [8, 9, 10, 11, 12, 13, 14, 15], '10.48.53.25': [16, 17, 18, 19, 20, 21, 22, 23], '10.48.53.162': [24, 25, 26, 27, 28, 29, 30, 31], '10.48.53.153': [32, 33, 34, 35, 36, 37, 38, 39], '10.48.52.101': [40, 41, 42, 43, 44, 45, 46, 47], '10.48.48.37': [48, 49, 50, 51, 52, 53, 54, 55], '10.48.48.32': [56, 57, 58, 59, 60, 61, 62, 63]}) 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:164:main] dist_world_size=64 10.48.48.92: [2026-03-22 02:39:29,641] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 10.48.48.92: [2026-03-22 02:39:29,642] [INFO] [launch.py:256:main] process 11786 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=0', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.92: [2026-03-22 02:39:29,642] [INFO] [launch.py:256:main] process 11787 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=1', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.92: [2026-03-22 02:39:29,643] [INFO] [launch.py:256:main] process 11788 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=2', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.92: [2026-03-22 02:39:29,643] [INFO] [launch.py:256:main] process 11789 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=3', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.92: [2026-03-22 02:39:29,644] [INFO] [launch.py:256:main] process 11790 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=4', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.92: [2026-03-22 02:39:29,644] [INFO] [launch.py:256:main] process 11791 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=5', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.92: [2026-03-22 02:39:29,645] [INFO] [launch.py:256:main] process 11792 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=6', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.92: [2026-03-22 02:39:29,645] [INFO] [launch.py:256:main] process 11793 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=7', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.25: [2026-03-22 02:39:42,008] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:39:42,151] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:42,731] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:43,163] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:139:main] 2 NCCL_VERSION=2.18.3-1 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:139:main] 2 NCCL_SOCKET_IFNAME=eth 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:139:main] 2 NCCL_IB_HCA=mlx5 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:139:main] 2 NCCL_IB_GID_INDEX=3 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:139:main] 2 NCCL_IB_DISABLE=0 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:146:main] WORLD INFO DICT: {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.25': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.162': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.153': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.52.101': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.37': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.32': [0, 1, 2, 3, 4, 5, 6, 7]} 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:152:main] nnodes=8, num_local_procs=8, node_rank=2 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(, {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [8, 9, 10, 11, 12, 13, 14, 15], '10.48.53.25': [16, 17, 18, 19, 20, 21, 22, 23], '10.48.53.162': [24, 25, 26, 27, 28, 29, 30, 31], '10.48.53.153': [32, 33, 34, 35, 36, 37, 38, 39], '10.48.52.101': [40, 41, 42, 43, 44, 45, 46, 47], '10.48.48.37': [48, 49, 50, 51, 52, 53, 54, 55], '10.48.48.32': [56, 57, 58, 59, 60, 61, 62, 63]}) 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:164:main] dist_world_size=64 10.48.53.25: [2026-03-22 02:39:43,886] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 10.48.53.25: [2026-03-22 02:39:43,887] [INFO] [launch.py:256:main] process 8927 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=0', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.25: [2026-03-22 02:39:43,887] [INFO] [launch.py:256:main] process 8928 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=1', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.25: [2026-03-22 02:39:43,888] [INFO] [launch.py:256:main] process 8929 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=2', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.25: [2026-03-22 02:39:43,888] [INFO] [launch.py:256:main] process 8930 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=3', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.25: [2026-03-22 02:39:43,889] [INFO] [launch.py:256:main] process 8931 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=4', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.25: [2026-03-22 02:39:43,889] [INFO] [launch.py:256:main] process 8932 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=5', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.25: [2026-03-22 02:39:43,890] [INFO] [launch.py:256:main] process 8933 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=6', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.25: [2026-03-22 02:39:43,891] [INFO] [launch.py:256:main] process 8934 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=7', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.153: [2026-03-22 02:39:43,991] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:44,318] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:44,330] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:139:main] 7 NCCL_VERSION=2.18.3-1 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:139:main] 7 NCCL_SOCKET_IFNAME=eth 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:139:main] 7 NCCL_IB_HCA=mlx5 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:139:main] 7 NCCL_IB_GID_INDEX=3 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:139:main] 7 NCCL_IB_DISABLE=0 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:146:main] WORLD INFO DICT: {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.25': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.162': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.153': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.52.101': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.37': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.32': [0, 1, 2, 3, 4, 5, 6, 7]} 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:152:main] nnodes=8, num_local_procs=8, node_rank=7 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(, {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [8, 9, 10, 11, 12, 13, 14, 15], '10.48.53.25': [16, 17, 18, 19, 20, 21, 22, 23], '10.48.53.162': [24, 25, 26, 27, 28, 29, 30, 31], '10.48.53.153': [32, 33, 34, 35, 36, 37, 38, 39], '10.48.52.101': [40, 41, 42, 43, 44, 45, 46, 47], '10.48.48.37': [48, 49, 50, 51, 52, 53, 54, 55], '10.48.48.32': [56, 57, 58, 59, 60, 61, 62, 63]}) 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:164:main] dist_world_size=64 10.48.48.32: [2026-03-22 02:39:44,391] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 10.48.48.32: [2026-03-22 02:39:44,392] [INFO] [launch.py:256:main] process 8978 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=0', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.32: [2026-03-22 02:39:44,392] [INFO] [launch.py:256:main] process 8979 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=1', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.32: [2026-03-22 02:39:44,393] [INFO] [launch.py:256:main] process 8980 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=2', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.32: [2026-03-22 02:39:44,393] [INFO] [launch.py:256:main] process 8981 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=3', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.32: [2026-03-22 02:39:44,393] [INFO] [launch.py:256:main] process 8982 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=4', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.32: [2026-03-22 02:39:44,394] [INFO] [launch.py:256:main] process 8983 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=5', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.32: [2026-03-22 02:39:44,394] [INFO] [launch.py:256:main] process 8984 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=6', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.32: [2026-03-22 02:39:44,395] [INFO] [launch.py:256:main] process 8985 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=7', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.48.37: [2026-03-22 02:39:44,737] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.37: [2026-03-22 02:39:44,749] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.52.101: [2026-03-22 02:39:45,924] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.153: [2026-03-22 02:39:46,429] [INFO] [launch.py:139:main] 4 NCCL_VERSION=2.18.3-1 10.48.53.153: [2026-03-22 02:39:46,430] [INFO] [launch.py:139:main] 4 NCCL_SOCKET_IFNAME=eth 10.48.53.153: [2026-03-22 02:39:46,430] [INFO] [launch.py:139:main] 4 NCCL_IB_HCA=mlx5 10.48.53.153: [2026-03-22 02:39:46,430] [INFO] [launch.py:139:main] 4 NCCL_IB_GID_INDEX=3 10.48.53.153: [2026-03-22 02:39:46,430] [INFO] [launch.py:139:main] 4 NCCL_IB_DISABLE=0 10.48.53.153: [2026-03-22 02:39:46,430] [INFO] [launch.py:146:main] WORLD INFO DICT: {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.25': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.162': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.53.153': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.52.101': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.37': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.48.32': [0, 1, 2, 3, 4, 5, 6, 7]} 10.48.53.153: [2026-03-22 02:39:46,430] [INFO] [launch.py:152:main] nnodes=8, num_local_procs=8, node_rank=4 10.48.53.153: [2026-03-22 02:39:46,430] [INFO] [launch.py:163:main] global_rank_mapping=defaultdict(, {'10.48.48.92': [0, 1, 2, 3, 4, 5, 6, 7], '10.48.54.166': [8, 9, 10, 11, 12, 13, 14, 15], '10.48.53.25': [16, 17, 18, 19, 20, 21, 22, 23], '10.48.53.162': [24, 25, 26, 27, 28, 29, 30, 31], '10.48.53.153': [32, 33, 34, 35, 36, 37, 38, 39], '10.48.52.101': [40, 41, 42, 43, 44, 45, 46, 47], '10.48.48.37': [48, 49, 50, 51, 52, 53, 54, 55], '10.48.48.32': [56, 57, 58, 59, 60, 61, 62, 63]}) 10.48.53.153: [2026-03-22 02:39:46,430] [INFO] [launch.py:164:main] dist_world_size=64 10.48.53.153: [2026-03-22 02:39:46,430] [INFO] [launch.py:168:main] Setting CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 10.48.53.153: [2026-03-22 02:39:46,431] [INFO] [launch.py:256:main] process 8949 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=0', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.153: [2026-03-22 02:39:46,431] [INFO] [launch.py:256:main] process 8950 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=1', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.153: [2026-03-22 02:39:46,432] [INFO] [launch.py:256:main] process 8951 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=2', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.153: [2026-03-22 02:39:46,432] [INFO] [launch.py:256:main] process 8952 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=3', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.153: [2026-03-22 02:39:46,432] [INFO] [launch.py:256:main] process 8953 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=4', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.153: [2026-03-22 02:39:46,433] [INFO] [launch.py:256:main] process 8954 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=5', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.153: [2026-03-22 02:39:46,433] [INFO] [launch.py:256:main] process 8955 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=6', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.53.153: [2026-03-22 02:39:46,434] [INFO] [launch.py:256:main] process 8956 spawned with command: ['/mmu_nlp_ssd/fukai07/basic_config/miniconda3/envs/swift_370/bin/python3.10', '-u', 'swift/cli/sft.py', '--local_rank=7', '--model', '/nlp_group/yueyang/model/Qwen3-32B', '--model_type', 'qwen3', '--torch_dtype', 'bfloat16', '--train_type', 'full', '--use_chat_template', 'true', '--dataset', '/mmu_nlp_hdd/luoxianzhen/sqzhou/reject_sampling/sft-dataset/CVE-V8.1.jsonl', '--per_device_train_batch_size', '1', '--data_seed', '42', '--weight_decay', '0.1', '--learning_rate', '1e-5', '--attn_impl', 'flash_attn', '--deepspeed', 'zero3', '--gradient_accumulation_steps', '4', '--dataloader_num_workers', '8', '--dataset_num_proc', '16', '--dataloader_pin_memory', 'true', '--dataloader_persistent_workers', 'true', '--dataloader_prefetch_factor', '2', '--warmup_ratio', '0.001', '--dataset_num_proc', '64', '--save_total_limit', '10', '--max_length', '65536', '--max_model_len', '65536', '--rope_scaling', 'yarn', '--sequence_parallel_size', '2', '--truncation_strategy', 'delete', '--split_dataset_ratio', '0', '--output_dir', '/mmu_nlp_hdd/luoxianzhen/models/sqzhou_cve-0322-CVE-V8.1-Qwen3-32B-lr1e-5-64kyarn_5epoch', '--eval_strategy', 'no', '--use_liger_kernel', 'true', '--use_hf', '--save_strategy', 'steps', '--save_steps', '100', '--num_train_epochs', '5', '--ddp_timeout', '180000', '--ignore_data_skip', 'true'] 10.48.52.101: [2026-03-22 02:39:46,690] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:46,994] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:47,122] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:47,157] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.52.101: [2026-03-22 02:39:47,165] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.162: [2026-03-22 02:39:47,685] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:47,939] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.52.101: [2026-03-22 02:39:47,948] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.52.101: [2026-03-22 02:39:48,062] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:48,419] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:48,442] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:48,484] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:48,487] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:48,497] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:48,491] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:48,659] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.37: [2026-03-22 02:39:48,671] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:48,722] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.162: [2026-03-22 02:39:48,746] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:48,852] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.37: [2026-03-22 02:39:48,865] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.37: [2026-03-22 02:39:48,898] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:48,912] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:48,999] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:49,031] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.162: [2026-03-22 02:39:49,372] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.162: [2026-03-22 02:39:49,385] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.162: [2026-03-22 02:39:49,450] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:49,487] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.52.101: [2026-03-22 02:39:49,496] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.54.166: [2026-03-22 02:39:49,518] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.54.166: [2026-03-22 02:39:49,541] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.54.166: [2026-03-22 02:39:49,557] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.54.166: [2026-03-22 02:39:49,588] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.54.166: [2026-03-22 02:39:49,589] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.54.166: [2026-03-22 02:39:49,632] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.54.166: [2026-03-22 02:39:49,636] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:49,928] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.52.101: [2026-03-22 02:39:49,939] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.52.101: [2026-03-22 02:39:49,953] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.52.101: [2026-03-22 02:39:49,962] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.52.101: [2026-03-22 02:39:49,992] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.52.101: [2026-03-22 02:39:50,002] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.162: [2026-03-22 02:39:50,010] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:50,046] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.52.101: [2026-03-22 02:39:50,054] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.162: [2026-03-22 02:39:50,062] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.52.101: [2026-03-22 02:39:50,056] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.52.101: [2026-03-22 02:39:50,065] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.162: [2026-03-22 02:39:50,113] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.162: [2026-03-22 02:39:50,122] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.162: [2026-03-22 02:39:50,133] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:50,338] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.37: [2026-03-22 02:39:50,351] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:50,394] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.92: [2026-03-22 02:39:50,406] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:50,406] [INFO] [comm.py:852:init_distributed] Initializing TorchBackend in DeepSpeed with backend nccl 10.48.53.162: [2026-03-22 02:39:50,473] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.162: [2026-03-22 02:39:50,485] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.37: [2026-03-22 02:39:50,793] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.37: [2026-03-22 02:39:50,800] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.37: [2026-03-22 02:39:50,805] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.37: [2026-03-22 02:39:50,812] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:50,957] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.37: [2026-03-22 02:39:50,978] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.37: [2026-03-22 02:39:50,995] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.37: [2026-03-22 02:39:51,002] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.37: [2026-03-22 02:39:51,019] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.162: [2026-03-22 02:39:51,344] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.162: [2026-03-22 02:39:51,357] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.54.166: [2026-03-22 02:39:51,370] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:51,382] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.54.166: [2026-03-22 02:39:51,462] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:51,475] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.54.166: [2026-03-22 02:39:51,478] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:51,491] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.54.166: [2026-03-22 02:39:51,539] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:51,552] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.54.166: [2026-03-22 02:39:51,552] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:51,553] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:51,565] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.54.166: [2026-03-22 02:39:51,566] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.54.166: [2026-03-22 02:39:51,579] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.54.166: [2026-03-22 02:39:51,590] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:51,943] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.162: [2026-03-22 02:39:51,964] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.162: [2026-03-22 02:39:51,979] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:52,053] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.162: [2026-03-22 02:39:52,109] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.162: [2026-03-22 02:39:52,124] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.162: [2026-03-22 02:39:52,173] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.162: [2026-03-22 02:39:52,185] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.162: [2026-03-22 02:39:52,186] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.162: [2026-03-22 02:39:52,198] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.162: [2026-03-22 02:39:52,229] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.162: [2026-03-22 02:39:52,243] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:52,280] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.92: [2026-03-22 02:39:52,304] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.92: [2026-03-22 02:39:52,340] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.92: [2026-03-22 02:39:52,366] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.92: [2026-03-22 02:39:52,760] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.92: [2026-03-22 02:39:52,774] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:53,775] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.92: [2026-03-22 02:39:53,788] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:53,982] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.92: [2026-03-22 02:39:53,997] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:54,250] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.92: [2026-03-22 02:39:54,269] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:54,339] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.92: [2026-03-22 02:39:54,351] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:54,353] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.92: [2026-03-22 02:39:54,370] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.92: [2026-03-22 02:39:54,401] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.92: [2026-03-22 02:39:54,415] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.25: [2026-03-22 02:40:04,216] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.25: [2026-03-22 02:40:04,580] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.25: [2026-03-22 02:40:04,584] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.25: [2026-03-22 02:40:04,599] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.25: [2026-03-22 02:40:04,617] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.25: [2026-03-22 02:40:04,654] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.25: [2026-03-22 02:40:04,717] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.25: [2026-03-22 02:40:04,736] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.25: [2026-03-22 02:40:05,655] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.25: [2026-03-22 02:40:05,665] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.25: [2026-03-22 02:40:06,190] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.25: [2026-03-22 02:40:06,200] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.25: [2026-03-22 02:40:06,229] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.25: [2026-03-22 02:40:06,239] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.25: [2026-03-22 02:40:06,266] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.25: [2026-03-22 02:40:06,275] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.25: [2026-03-22 02:40:06,294] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.25: [2026-03-22 02:40:06,304] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.25: [2026-03-22 02:40:06,326] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.25: [2026-03-22 02:40:06,335] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.25: [2026-03-22 02:40:06,348] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.25: [2026-03-22 02:40:06,353] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.25: [2026-03-22 02:40:06,358] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.25: [2026-03-22 02:40:06,362] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.32: [2026-03-22 02:40:06,388] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.32: [2026-03-22 02:40:06,388] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.32: [2026-03-22 02:40:06,389] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.32: [2026-03-22 02:40:06,389] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.32: [2026-03-22 02:40:06,389] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.32: [2026-03-22 02:40:06,389] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.32: [2026-03-22 02:40:06,389] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.32: [2026-03-22 02:40:06,390] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.48.32: [2026-03-22 02:40:08,190] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:40:08,205] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.32: [2026-03-22 02:40:08,379] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:40:08,392] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.32: [2026-03-22 02:40:08,422] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:40:08,425] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:40:08,434] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.32: [2026-03-22 02:40:08,434] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:40:08,436] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:40:08,437] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.32: [2026-03-22 02:40:08,446] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:40:08,447] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.32: [2026-03-22 02:40:08,450] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.32: [2026-03-22 02:40:08,459] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.48.32: [2026-03-22 02:40:08,472] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.48.32: [2026-03-22 02:40:08,484] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.153: [2026-03-22 02:40:08,520] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.153: [2026-03-22 02:40:08,520] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.153: [2026-03-22 02:40:08,520] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.153: [2026-03-22 02:40:08,522] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.153: [2026-03-22 02:40:08,566] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.153: [2026-03-22 02:40:08,573] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.153: [2026-03-22 02:40:08,581] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.153: [2026-03-22 02:40:08,606] [INFO] [real_accelerator.py:254:get_accelerator] Setting ds_accelerator to cuda (auto detect) 10.48.53.153: [2026-03-22 02:40:10,648] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.153: [2026-03-22 02:40:10,661] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.153: [2026-03-22 02:40:10,665] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.153: [2026-03-22 02:40:10,678] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.153: [2026-03-22 02:40:10,688] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.153: [2026-03-22 02:40:10,701] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.153: [2026-03-22 02:40:10,721] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.153: [2026-03-22 02:40:10,722] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.153: [2026-03-22 02:40:10,733] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.153: [2026-03-22 02:40:10,735] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.153: [2026-03-22 02:40:10,736] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.153: [2026-03-22 02:40:10,742] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.153: [2026-03-22 02:40:10,755] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.153: [2026-03-22 02:40:10,758] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.153: [2026-03-22 02:40:10,793] [INFO] [logging.py:107:log_dist] [Rank -1] [TorchCheckpointEngine] Initialized with serialization = False 10.48.53.153: [2026-03-22 02:40:10,808] [INFO] [comm.py:821:init_distributed] cdb=None 10.48.53.25: [2026-03-22 02:40:20,897] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.25: [2026-03-22 02:40:20,934] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.25: [2026-03-22 02:40:20,934] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.25: [2026-03-22 02:40:20,934] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.25: [2026-03-22 02:40:20,936] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.25: [2026-03-22 02:40:20,939] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.25: [2026-03-22 02:40:20,950] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.25: [2026-03-22 02:40:20,952] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.37: [2026-03-22 02:40:20,986] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.37: [2026-03-22 02:40:20,993] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.54.166: [2026-03-22 02:40:20,988] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.37: [2026-03-22 02:40:20,995] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.37: [2026-03-22 02:40:20,995] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.54.166: [2026-03-22 02:40:20,994] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.37: [2026-03-22 02:40:21,003] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.92: [2026-03-22 02:40:20,999] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.37: [2026-03-22 02:40:21,006] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.37: [2026-03-22 02:40:21,009] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.54.166: [2026-03-22 02:40:21,005] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.54.166: [2026-03-22 02:40:21,009] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.54.166: [2026-03-22 02:40:21,012] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.92: [2026-03-22 02:40:21,014] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.54.166: [2026-03-22 02:40:21,015] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.162: [2026-03-22 02:40:21,023] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.37: [2026-03-22 02:40:21,026] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.32: [2026-03-22 02:40:21,020] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.32: [2026-03-22 02:40:21,020] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.32: [2026-03-22 02:40:21,022] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.92: [2026-03-22 02:40:21,025] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.54.166: [2026-03-22 02:40:21,026] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.162: [2026-03-22 02:40:21,032] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.32: [2026-03-22 02:40:21,027] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.92: [2026-03-22 02:40:21,029] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.92: [2026-03-22 02:40:21,029] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.32: [2026-03-22 02:40:21,029] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.32: [2026-03-22 02:40:21,030] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.153: [2026-03-22 02:40:21,031] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.162: [2026-03-22 02:40:21,038] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.32: [2026-03-22 02:40:21,031] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.92: [2026-03-22 02:40:21,035] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.162: [2026-03-22 02:40:21,041] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.162: [2026-03-22 02:40:21,043] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.54.166: [2026-03-22 02:40:21,039] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.162: [2026-03-22 02:40:21,048] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.153: [2026-03-22 02:40:21,044] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.153: [2026-03-22 02:40:21,047] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.153: [2026-03-22 02:40:21,047] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.92: [2026-03-22 02:40:21,048] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.153: [2026-03-22 02:40:21,048] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.92: [2026-03-22 02:40:21,048] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.32: [2026-03-22 02:40:21,049] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.153: [2026-03-22 02:40:21,053] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.162: [2026-03-22 02:40:21,070] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.153: [2026-03-22 02:40:21,070] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.153: [2026-03-22 02:40:21,070] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.53.162: [2026-03-22 02:40:21,081] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.52.101: [2026-03-22 02:40:21,115] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.52.101: [2026-03-22 02:40:21,117] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.52.101: [2026-03-22 02:40:21,117] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.52.101: [2026-03-22 02:40:21,118] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.52.101: [2026-03-22 02:40:21,119] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.52.101: [2026-03-22 02:40:21,142] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.52.101: [2026-03-22 02:40:21,144] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.52.101: [2026-03-22 02:40:21,155] [INFO] [config.py:684:__init__] Config mesh_device None world_size = 64 10.48.48.92: [2026-03-22 02:40:21,707] [INFO] [partition_parameters.py:366:__exit__] finished initializing model - num_params = 707, num_elems = 32.76B 10.48.48.92: Parameter Offload - Persistent parameters statistics: param_count = 257, numel = 676864