Upload 13 files

Browse files

Files changed (14) hide show

.gitattributes +1 -0
LLARA-passage-paddle/added_tokens.json +1 -0
LLARA-passage-paddle/all_results.json +9 -0
LLARA-passage-paddle/config.json +40 -0
LLARA-passage-paddle/lora_config.json +35 -0
LLARA-passage-paddle/lora_model_state.pdparams +3 -0
LLARA-passage-paddle/runs/Nov01_14-28-20_b63e119a1648/vdlrecords.1762007362.log +0 -0
LLARA-passage-paddle/sentencepiece.bpe.model +3 -0
LLARA-passage-paddle/special_tokens_map.json +1 -0
LLARA-passage-paddle/tokenizer_config.json +1 -0
LLARA-passage-paddle/train_LLARA-passage-paddle_mmarco.jsonl.log +777 -0
LLARA-passage-paddle/train_results.json +9 -0
LLARA-passage-paddle/trainer_state.json +15 -0
LLARA-passage-paddle/training_args.bin +3 -0

.gitattributes CHANGED Viewed

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+LLARA-passage-paddle/lora_model_state.pdparams filter=lfs diff=lfs merge=lfs -text

LLARA-passage-paddle/added_tokens.json ADDED Viewed

	@@ -0,0 +1 @@


1	+ {"<unk>": 0, "<s>": 1, "</s>": 2, "<s9>": 32000, "<s12>": 32001, "<s7>": 32002, "<s14>": 32003, "<s3>": 32004, "<s6>": 32005, "<s15>": 32006, "<s5>": 32007, "<s1>": 32008, "<s10>": 32009, "<s13>": 32010, "<s2>": 32011, "<s11>": 32012, "<s4>": 32013, "<s8>": 32014, "<s16>": 32015}

LLARA-passage-paddle/all_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "cpu_available_memory": 938.47,
+    "cpu_used_memory": 62.38,
+    "progress_or_epoch": 0.064,
+    "train_loss": 0.7623166245222092,
+    "train_runtime": 40117.9587,
+    "train_samples_per_second": 0.3191,
+    "train_steps_per_second": 0.0025
+}

LLARA-passage-paddle/config.json ADDED Viewed

	@@ -0,0 +1,40 @@

+{
+  "_name_or_path": "LLARA-passage",
+  "alibi": false,
+  "architectures": [
+    "LlamaModel"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dpo_config": null,
+  "dtype": "float16",
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "immediate_clear_past_key_value": false,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "long_sequence_init_args": {},
+  "long_sequence_strategy_name": null,
+  "long_sequence_strategy_type": null,
+  "max_position_embeddings": 4096,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "paddlenlp_version": "3.0.0b4.post20251101",
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_scaling_factor": 1.0,
+  "rope_scaling_type": null,
+  "rope_theta": 10000.0,
+  "seq_length": 2048,
+  "tie_word_embeddings": false,
+  "use_flash_attention_for_generation": false,
+  "use_last_token_for_generation": false,
+  "use_long_sequence_strategies": false,
+  "vocab_size": 32016
+}

LLARA-passage-paddle/lora_config.json ADDED Viewed

	@@ -0,0 +1,35 @@

+{
+  "base_model_name_or_path": null,
+  "do_qat": false,
+  "dtype": "float16",
+  "enable_lora_list": null,
+  "head_dim": null,
+  "lora_alpha": 64,
+  "lora_dropout": 0.1,
+  "lora_plus_scale": 1.0,
+  "lora_use_mixer": false,
+  "loraga": false,
+  "lorapro": false,
+  "merge_weights": false,
+  "mixer_num": 1,
+  "nola": false,
+  "nola_basis_num": 1,
+  "pissa": false,
+  "r": 32,
+  "rslora": false,
+  "scaling": 2.0,
+  "target_modules": [
+    ".*q_proj$",
+    ".*k_proj$",
+    ".*v_proj$",
+    ".*o_proj$",
+    ".*down_proj$",
+    ".*up_proj$",
+    ".*gate_proj$"
+  ],
+  "tensor_parallel_degree": 1,
+  "trainable_bias": null,
+  "trainable_modules": null,
+  "use_mora": false,
+  "use_quick_lora": false
+}

LLARA-passage-paddle/lora_model_state.pdparams ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:d4882e10ba10a202cfba7ec77989652354193bcb6c91d24fe13616fbb258ffe4
+size 479881234

LLARA-passage-paddle/runs/Nov01_14-28-20_b63e119a1648/vdlrecords.1762007362.log ADDED Viewed

Binary file (78.2 kB). View file

LLARA-passage-paddle/sentencepiece.bpe.model ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

LLARA-passage-paddle/special_tokens_map.json ADDED Viewed

	@@ -0,0 +1 @@

+ {"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": false}, "pad_token": "<unk>", "additional_special_tokens": ["<s9>", "<s12>", "<s7>", "<s14>", "<s3>", "<s6>", "<s15>", "<s5>", "<s1>", "<s10>", "<s13>", "<s2>", "<s11>", "<s4>", "<s8>", "<s16>"]}

LLARA-passage-paddle/tokenizer_config.json ADDED Viewed

	@@ -0,0 +1 @@

+ {"unk_token": "<unk>", "bos_token": "<s>", "eos_token": "</s>", "add_bos_token": true, "add_eos_token": false, "sp_model_kwargs": {}, "decode_with_prefix_space": false, "add_prefix_space": true, "added_tokens_decoder": {"0": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "1": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "2": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32000": {"content": "<s9>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32001": {"content": "<s12>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32002": {"content": "<s7>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32003": {"content": "<s14>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32004": {"content": "<s3>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32005": {"content": "<s6>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32006": {"content": "<s15>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32007": {"content": "<s5>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32008": {"content": "<s1>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32009": {"content": "<s10>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32010": {"content": "<s13>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32011": {"content": "<s2>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32012": {"content": "<s11>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32013": {"content": "<s4>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32014": {"content": "<s8>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, "32015": {"content": "<s16>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}}, "additional_special_tokens": ["<s9>", "<s12>", "<s7>", "<s14>", "<s3>", "<s6>", "<s15>", "<s5>", "<s1>", "<s10>", "<s13>", "<s2>", "<s11>", "<s4>", "<s8>", "<s16>"], "clean_up_tokenization_spaces": false, "legacy": false, "model_max_length": 1000000000000000019884624838656, "pad_token": "<unk>", "padding_side": "left", "spaces_between_special_tokens": false, "use_default_system_prompt": true, "special_tokens_map_file": "/mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle/special_tokens_map.json", "name_or_path": "/mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle", "from_slow": true, "_from_auto": true, "added_tokens_file": "/mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle/added_tokens.json", "tokenizer_class": "LlamaTokenizer"}

LLARA-passage-paddle/train_LLARA-passage-paddle_mmarco.jsonl.log ADDED Viewed

	@@ -0,0 +1,777 @@

+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md
+  warnings.warn(warning_message)
+LAUNCH INFO 2025-11-01 14:28:14,900 -----------  Configuration  ----------------------
+LAUNCH INFO 2025-11-01 14:28:14,900 auto_cluster_config: 0
+LAUNCH INFO 2025-11-01 14:28:14,900 auto_parallel_config: None
+LAUNCH INFO 2025-11-01 14:28:14,900 auto_tuner_json: None
+LAUNCH INFO 2025-11-01 14:28:14,900 devices: 0,1,2,3
+LAUNCH INFO 2025-11-01 14:28:14,900 elastic_level: -1
+LAUNCH INFO 2025-11-01 14:28:14,900 elastic_timeout: 30
+LAUNCH INFO 2025-11-01 14:28:14,900 enable_gpu_log: True
+LAUNCH INFO 2025-11-01 14:28:14,900 gloo_port: 6767
+LAUNCH INFO 2025-11-01 14:28:14,900 host: None
+LAUNCH INFO 2025-11-01 14:28:14,900 ips: None
+LAUNCH INFO 2025-11-01 14:28:14,900 job_id: default
+LAUNCH INFO 2025-11-01 14:28:14,918 legacy: False
+LAUNCH INFO 2025-11-01 14:28:14,918 log_dir: log
+LAUNCH INFO 2025-11-01 14:28:14,918 log_level: INFO
+LAUNCH INFO 2025-11-01 14:28:14,918 log_overwrite: False
+LAUNCH INFO 2025-11-01 14:28:14,918 master: None
+LAUNCH INFO 2025-11-01 14:28:14,918 max_restart: 3
+LAUNCH INFO 2025-11-01 14:28:14,918 nnodes: 1
+LAUNCH INFO 2025-11-01 14:28:14,918 nproc_per_node: None
+LAUNCH INFO 2025-11-01 14:28:14,918 rank: -1
+LAUNCH INFO 2025-11-01 14:28:14,918 run_mode: collective
+LAUNCH INFO 2025-11-01 14:28:14,918 server_num: None
+LAUNCH INFO 2025-11-01 14:28:14,920 servers:
+LAUNCH INFO 2025-11-01 14:28:14,920 sort_ip: False
+LAUNCH INFO 2025-11-01 14:28:14,920 start_port: 6070
+LAUNCH INFO 2025-11-01 14:28:14,920 trainer_num: None
+LAUNCH INFO 2025-11-01 14:28:14,920 trainers:
+LAUNCH INFO 2025-11-01 14:28:14,920 training_script: train.py
+LAUNCH INFO 2025-11-01 14:28:14,921 training_script_args: ['--do_train', '--query_instruction_for_retrieval', 'query: ', '--passage_instruction_for_retrieval', '', '--model_name_or_path', '/mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle', '--output_dir', 'tmp_train_mmarco.jsonl/LLARA-passage-paddle', '--save_steps', '999999999', '--train_data', './data/mmarco.jsonl', '--fp16_opt_level', 'O2', '--fp16', '--per_device_train_batch_size', '1', '--gradient_accumulation_steps', '32', '--recompute', '--train_group_size', '4', '--learning_rate', '1e-4', '--query_max_len', '128', '--passage_max_len', '4096', '--num_train_epochs', '1', '--logging_steps', '1', '--overwrite_output_dir', '--negatives_cross_device', '--warmup_steps', '10', '--max_steps', '100', '--do_train', '--fine_tune_type', 'lora', '--sentence_pooling_method', 'last_8', '--sharding', 'stage3 offload', '--use_flash_attention', '--temperature', '0.01']
+LAUNCH INFO 2025-11-01 14:28:14,921 with_gloo: 1
+LAUNCH INFO 2025-11-01 14:28:14,922 --------------------------------------------------
+LAUNCH INFO 2025-11-01 14:28:14,928 Job: default, mode collective, replicas 1[1:1], elastic False
+LAUNCH INFO 2025-11-01 14:28:14,967 Run Pod: yonmjh, replicas 4, status ready
+LAUNCH INFO 2025-11-01 14:28:15,454 Watching Pod: yonmjh, replicas 4, status running
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md
+  warnings.warn(warning_message)
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/_distutils_hack/__init__.py:30: UserWarning: Setuptools is replacing distutils. Support for replacing an already imported distutils is deprecated. In the future, this condition will fail. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/mnt/141nfs/lizhuoqun/PaddleNLP_1022/PaddleNLP/paddlenlp/trainer/training_args.py:1271: UserWarning: `offload` is not supported NOW!
+  warnings.warn("`offload` is not supported NOW!")
+[2025-11-01 14:28:20,247] [    INFO] distributed_strategy.py:335 - distributed strategy initialized
+======================= Modified FLAGS detected =======================
+FLAGS(name='FLAGS_enable_pir_in_executor', current_value=True, default_value=False)
+FLAGS(name='FLAGS_selected_gpus', current_value='0', default_value='')
+FLAGS(name='FLAGS_cublas_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cublas/lib', default_value='')
+FLAGS(name='FLAGS_cusolver_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cusolver/lib', default_value='')
+FLAGS(name='FLAGS_nvidia_package_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia', default_value='')
+FLAGS(name='FLAGS_cudnn_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cudnn/lib', default_value='')
+FLAGS(name='FLAGS_cusparse_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cusparse/lib', default_value='')
+FLAGS(name='FLAGS_curand_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/curand/lib', default_value='')
+FLAGS(name='FLAGS_cuda_cccl_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cuda_cccl/include/', default_value='')
+FLAGS(name='FLAGS_nccl_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/nccl/lib', default_value='')
+FLAGS(name='FLAGS_cupti_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cuda_cupti/lib', default_value='')
+=======================================================================
+I1101 14:28:20.248835 60039 tcp_store.cc:336] create libuv server at port: 63428
+I1101 14:28:20.249553 60039 tcp_utils.cc:132] Successfully connected to 172.17.0.5:63428
+I1101 14:28:23.628891 60039 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
+I1101 14:28:23.628948 60039 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
+[2025-11-01 14:28:23,629] [    INFO] topology.py:526 - Total 4 pipe comm group(s) create successfully!
+W1101 14:28:23.630813 60039 gpu_resources.cc:114] Please NOTE: device: 0, GPU Compute Capability: 8.0, Driver API Version: 12.8, Runtime API Version: 12.8
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/distributed/communication/group.py:145: UserWarning: Current global rank 0 is not in group _default_pg10
+  warnings.warn(
+[2025-11-01 14:28:23,633] [    INFO] topology.py:526 - Total 4 data comm group(s) create successfully!
+[2025-11-01 14:28:23,635] [    INFO] topology.py:526 - Total 4 model comm group(s) create successfully!
+I1101 14:28:23.635226 60039 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
+I1101 14:28:23.635236 60039 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
+[2025-11-01 14:28:23,635] [    INFO] topology.py:526 - Total 1 sharding comm group(s) create successfully!
+I1101 14:28:23.635305 60039 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
+I1101 14:28:23.635313 60039 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
+[2025-11-01 14:28:23,635] [    INFO] topology.py:440 - HybridParallelInfo: rank_id: 0, mp_degree: 1, sharding_degree: 4, pp_degree: 1, dp_degree: 1, sep_degree: 1, mp_group: [0],  sharding_group: [0, 1, 2, 3], pp_group: [0], dp_group: [0], sep:group: None, check/clip group: [0, 1, 2, 3]
+[32m[2025-11-01 14:28:23,635] [    INFO][0m -     +==============================================================================+
+    |                                                                              |
+    |                         DistributedStrategy Overview                         |
+    |                                                                              |
+    +==============================================================================+
+    |                        a_sync=True <-> a_sync_configs                        |
+    +------------------------------------------------------------------------------+
+    |                               k_steps                    -1                  |
+    |                     max_merge_var_num                    1                   |
+    |                       send_queue_size                    16                  |
+    |               independent_recv_thread                  False                 |
+    |         min_send_grad_num_before_recv                    1                   |
+    |                      thread_pool_size                    1                   |
+    |                       send_wait_times                    1                   |
+    |               runtime_split_send_recv                  False                 |
+    |                        launch_barrier                   True                 |
+    |             heter_worker_device_guard                   cpu                  |
+    |                        lr_decay_steps                    10                  |
+    |                            use_ps_gpu                    0                   |
+    |                         use_gpu_graph                    0                   |
+    +==============================================================================+
+    |                    Environment Flags, Communication Flags                    |
+    +------------------------------------------------------------------------------+
+    |                                  mode                    1                   |
+    |                               elastic                  False                 |
+    |                                  auto                  False                 |
+    |                   sync_nccl_allreduce                   True                 |
+    |                         nccl_comm_num                    1                   |
+    |            use_hierarchical_allreduce                  False                 |
+    |   hierarchical_allreduce_inter_nranks                    1                   |
+    |                       sync_batch_norm                  False                 |
+    |                   fuse_all_reduce_ops                   True                 |
+    |                  fuse_grad_size_in_MB                    32                  |
+    |              fuse_grad_size_in_TFLOPS                   50.0                 |
+    |               cudnn_exhaustive_search                  False                 |
+    |             conv_workspace_size_limit                   512                  |
+    |    cudnn_batchnorm_spatial_persistent                  False                 |
+    |                        fp16_allreduce                  False                 |
+    |               last_comm_group_size_MB                   1.0                  |
+    |                find_unused_parameters                  False                 |
+    |            without_graph_optimization                   True                 |
+    |                 fuse_grad_size_in_num                    8                   |
+    |                 calc_comm_same_stream                  False                 |
+    |                                   asp                  False                 |
+    |                       fuse_grad_merge                  False                 |
+    |                             semi_auto                  False                 |
+    |                            adam_d2sum                  False                 |
+    |                           auto_search                  False                 |
+    |                        heter_ccl_mode                  False                 |
+    |                         is_fl_ps_mode                  False                 |
+    |                      with_coordinator                  False                 |
+    |                            split_data                   True                 |
+    |                  downpour_table_param                    []                  |
+    |                       fs_client_param                                        |
+    +==============================================================================+
+    |                                Build Strategy                                |
+    +------------------------------------------------------------------------------+
+    |              fuse_elewise_add_act_ops                  False                 |
+    |                       fuse_bn_act_ops                  False                 |
+    |              fuse_relu_depthwise_conv                  False                 |
+    |                    fuse_broadcast_ops                  False                 |
+    |                fuse_all_optimizer_ops                  False                 |
+    |                        enable_inplace                  False                 |
+    |     enable_backward_optimizer_op_deps                   True                 |
+    |                 cache_runtime_context                  False                 |
+    |                   fuse_bn_add_act_ops                   True                 |
+    |                    enable_auto_fusion                  False                 |
+    |                          enable_addto                  False                 |
+    |              allow_cuda_graph_capture                  False                 |
+    |                       reduce_strategy                    0                   |
+    |                    fuse_gemm_epilogue                  False                 |
+    |                   debug_graphviz_path                                        |
+    |                       fused_attention                  False                 |
+    |                     fused_feedforward                  False                 |
+    |            fuse_dot_product_attention                  False                 |
+    |                          fuse_resunit                  False                 |
+    +==============================================================================+
+[0m
+[32m[2025-11-01 14:28:23,635] [    INFO][0m - The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).[0m
+[33m[2025-11-01 14:28:23,636] [ WARNING][0m - Process rank: 0, device: gpu, distributed training: True, 16-bits training: True[0m
+[32m[2025-11-01 14:28:23,636] [    INFO][0m - Training/evaluation parameters RetrieverTrainingArguments(
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+amp_custom_black_list=None,
+amp_custom_white_list=None,
+amp_master_grad=False,
+aoa_config=None,
+auto_parallel_resume_form_hybrid_parallel=False,
+bf16=False,
+bf16_full_eval=False,
+ckpt_quant_stage=O0,
+context_parallel_degree=1,
+count_trained_tokens=False,
+data_parallel_config=,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_shuffle=True,
+ddp_find_unused_parameters=None,
+device=gpu,
+disable_tqdm=False,
+distributed_dataloader=False,
+do_eval=False,
+do_export=False,
+do_predict=False,
+do_train=True,
+enable_auto_parallel=False,
+enable_zero_cost_checkpoint=False,
+eval_accumulation_steps=None,
+eval_steps=None,
+evaluation_strategy=IntervalStrategy.NO,
+expert_max_capacity=4294967296,
+expert_min_capacity=1,
+expert_parallel_degree=1,
+expert_tensor_parallel_degree=1,
+fine_tune_type=lora,
+fix_position_embedding=False,
+flash_device_save_steps=0,
+flatten_param_grads=False,
+force_reshard_pp=False,
+fp16=True,
+fp16_full_eval=False,
+fp16_opt_level=O2,
+fuse_sequence_parallel_allreduce=False,
+gradient_accumulation_steps=32,
+greater_is_better=None,
+hybrid_parallel_topo_order=sharding_first,
+ignore_data_skip=False,
+ignore_load_lr_and_optim=False,
+ignore_save_lr_and_optim=False,
+label_names=None,
+lazy_data_processing=True,
+learning_rate=0.0001,
+load_best_model_at_end=False,
+load_checkpoint_format=None,
+load_sharded_model=False,
+load_sharded_model_remap_parameter_name=False,
+local_rank=0,
+log_on_each_node=True,
+logging_dir=tmp_train_mmarco.jsonl/LLARA-passage-paddle/runs/Nov01_14-28-20_b63e119a1648,
+logging_first_step=False,
+logging_steps=1,
+logging_strategy=IntervalStrategy.STEPS,
+lr_end=1e-07,
+lr_scheduler_type=SchedulerType.LINEAR,
+margin=0.2,
+matryoshka_dims=[64, 128, 256, 512, 768],
+matryoshka_loss_weights=[1, 1, 1, 1, 1],
+max_evaluate_steps=-1,
+max_grad_norm=1.0,
+max_steps=100,
+metric_for_best_model=None,
+metrics_output_path=None,
+min_lr=0.0,
+minimum_eval_times=None,
+nccl_comm_group_config=None,
+negatives_cross_device=True,
+no_cuda=False,
+num_cycles=0.5,
+num_train_epochs=1.0,
+offload_optim=False,
+optim=OptimizerNames.ADAMW,
+ordered_save_group_size=0,
+output_dir=tmp_train_mmarco.jsonl/LLARA-passage-paddle,
+output_signal_dir=tmp_train_mmarco.jsonl/LLARA-passage-paddle,
+overwrite_output_dir=True,
+pad_token_id=0,
+past_index=-1,
+pdc_download_ckpt=False,
+pdc_download_timeout=300,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+pipeline_parallel_config=,
+pipeline_parallel_degree=1,
+power=1.0,
+prediction_loss_only=False,
+recompute=True,
+refined_recompute={},
+release_grads=False,
+remove_unused_columns=True,
+report_to=['visualdl'],
+resume_from_checkpoint=None,
+run_name=tmp_train_mmarco.jsonl/LLARA-passage-paddle,
+save_checkpoint_format=None,
+save_on_each_node=False,
+save_rng_states=True,
+save_sharded_model=False,
+save_sharding_stage1_model_include_freeze_params=False,
+save_steps=999999999,
+save_strategy=IntervalStrategy.STEPS,
+save_tokenizer=True,
+save_total_limit=None,
+scale_loss=32768,
+seed=42,
+sentence_pooling_method=last_8,
+sep_parallel_degree=1,
+sequence_parallel=False,
+sequence_parallel_config=,
+sharded_model_from_ema=False,
+sharding=[<ShardingOption.FULL_SHARD: 'stage3'>, <ShardingOption.OFFLOAD: 'offload'>],
+sharding_comm_buffer_size_MB=-1,
+sharding_degree=-1,
+sharding_offload_opt_buffersize_GB=-1,
+sharding_parallel_config=,
+sharding_parallel_degree=4,
+sharding_parallel_mesh_dimension=dp,
+skip_data_intervals=None,
+skip_memory_metrics=True,
+skip_profile_timer=True,
+split_inputs_sequence_dim=True,
+split_norm_comm=False,
+temperature=0.01,
+tensor_parallel_config=,
+tensor_parallel_degree=1,
+tensorwise_offload_optimizer=False,
+to_static=False,
+unified_checkpoint=False,
+unified_checkpoint_config=,
+use_async_save=False,
+use_expert_parallel=False,
+use_inbatch_neg=False,
+use_lowprecision_moment=False,
+use_matryoshka=False,
+wandb_api_key=None,
+wandb_http_proxy=None,
+warmup_ratio=0.0,
+warmup_steps=10,
+weight_decay=0.0,
+zcc_ema_interval=1,
+zcc_ema_loss_threshold=None,
+zcc_pipeline_hooks_capacity_usage=0.6,
+zcc_save_ema_coef=None,
+zcc_workers_num=3,
+)[0m
+[32m[2025-11-01 14:28:23,636] [    INFO][0m - Model parameters ModelArguments(model_name_or_path='/mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle', tokenizer_name=None, normalized=True, use_flash_attention=True)[0m
+[32m[2025-11-01 14:28:23,636] [    INFO][0m - Data parameters DataArguments(train_data='./data/mmarco.jsonl', train_group_size=4, query_max_len=128, passage_max_len=4096, max_example_num_per_dataset=100000000, query_instruction_for_retrieval='query: ', passage_instruction_for_retrieval='')[0m
+[32m[2025-11-01 14:28:23,638] [    INFO][0m - The global seed is set to 42, local seed is set to 46 and random seed is set to 42.[0m
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+Ignored unknown kwarg option __type
+[32m[2025-11-01 14:28:23,650] [    INFO][0m - We are using <class 'paddlenlp.transformers.llama.modeling.LlamaModel'> to load '/mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle'.[0m
+[32m[2025-11-01 14:28:23,720] [    INFO][0m - Loading configuration file /mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle/config.json[0m
+[32m[2025-11-01 14:28:23,721] [    INFO][0m - Loading weights file /mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle/model_state.pdparams.index.json[0m
+file: /mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle/model_state.pdparams.index.json is paddle weight.
+Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]
+Loading checkpoint shards:  50%|█████     | 1/2 [00:17<00:17, 17.39s/it]
+Loading checkpoint shards: 100%|██████████| 2/2 [00:23<00:00, 10.53s/it]
+Loading checkpoint shards: 100%|██████████| 2/2 [00:23<00:00, 11.56s/it]
+[32m[2025-11-01 14:28:47,257] [    INFO][0m - All model checkpoint weights were used when initializing LlamaModel.
+[0m
+[32m[2025-11-01 14:28:47,260] [    INFO][0m - All the weights of LlamaModel were initialized from the model checkpoint at /mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaModel for predictions without further training.[0m
+[32m[2025-11-01 14:28:47,263] [    INFO][0m - Loading configuration file /mnt/141nfs/lizhuoqun/hf_models/LLARA-passage-paddle/config.json[0m
+[33m[2025-11-01 14:28:47,643] [ WARNING][0m - Reset tensor_parallel_degree of lora_config to 1.[0m
+[32m[2025-11-01 14:28:47,643] [    INFO][0m - Mark only lora and trainable_module as trainable.[0m
+[35m[2025-11-01 14:28:47,672] [   DEBUG][0m - Frozen parameters: 1.98e+10 || Trainable parameters:2.40e+08 || Total parameters:2.01e+10|| Trainable:1.20%[0m
+[32m[2025-11-01 14:28:47,717] [    INFO][0m - The global seed is set to 42, local seed is set to 46 and random seed is set to 42.[0m
+[32m[2025-11-01 14:28:47,778] [    INFO][0m - max_steps is given, it will override any value given in num_train_epochs[0m
+[32m[2025-11-01 14:28:47,778] [    INFO][0m - Using half precision[0m
+[35m[2025-11-01 14:28:47,802] [   DEBUG][0m - ============================================================[0m
+[35m[2025-11-01 14:28:47,802] [   DEBUG][0m -     Training Configuration Arguments    [0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - paddle commit id              : dc312219f1498234f178a3b6a93e9f1f4ddebcb8[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - paddlenlp commit id           : 51787ba815d78521c373d8c08f72cc3a10a425da.dirty[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - _no_sync_in_gradient_accumulation: True[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - adam_beta1                    : 0.9[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - adam_beta2                    : 0.999[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - adam_epsilon                  : 1e-08[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - amp_custom_black_list         : None[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - amp_custom_white_list         : None[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - amp_master_grad               : False[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - aoa_config                    : None[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - auto_parallel_resume_form_hybrid_parallel: False[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - bf16                          : False[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - bf16_full_eval                : False[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - ckpt_quant_stage              : O0[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - context_parallel_degree       : 1[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - context_parallel_rank         : 0[0m
+[35m[2025-11-01 14:28:47,803] [   DEBUG][0m - count_trained_tokens          : False[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - current_device                : gpu:0[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - data_parallel_config          : [0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - data_parallel_degree          : 1[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - data_parallel_rank            : 0[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - dataloader_drop_last          : False[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - dataloader_num_workers        : 0[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - dataloader_shuffle            : True[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - dataset_rank                  : 0[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - dataset_world_size            : 4[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - ddp_find_unused_parameters    : None[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - device                        : gpu[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - disable_tqdm                  : False[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - distributed_dataloader        : False[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - do_eval                       : False[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - do_export                     : False[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - do_predict                    : False[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - do_train                      : True[0m
+[35m[2025-11-01 14:28:47,804] [   DEBUG][0m - enable_auto_parallel          : False[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - enable_zero_cost_checkpoint   : False[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - eval_accumulation_steps       : None[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - eval_batch_size               : 8[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - eval_steps                    : None[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - evaluation_strategy           : IntervalStrategy.NO[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - expert_max_capacity           : 4294967296[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - expert_min_capacity           : 1[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - expert_parallel_degree        : 1[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - expert_parallel_rank          : 0[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - expert_tensor_parallel_degree : 1[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - fine_tune_type                : lora[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - fix_position_embedding        : False[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - flash_device_save_steps       : 0[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - flatten_param_grads           : False[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - force_reshard_pp              : False[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - fp16                          : True[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - fp16_full_eval                : False[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - fp16_opt_level                : O2[0m
+[35m[2025-11-01 14:28:47,805] [   DEBUG][0m - fuse_sequence_parallel_allreduce: False[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - gradient_accumulation_steps   : 32[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - greater_is_better             : None[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - hybrid_parallel_topo_order    : sharding_first[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - ignore_data_skip              : False[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - ignore_load_lr_and_optim      : False[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - ignore_save_lr_and_optim      : False[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - label_names                   : None[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - lazy_data_processing          : True[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - learning_rate                 : 0.0001[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - load_best_model_at_end        : False[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - load_checkpoint_format        : None[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - load_sharded_model            : False[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - load_sharded_model_remap_parameter_name: False[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - local_process_index           : 0[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - local_rank                    : 0[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - log_level                     : -1[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - log_level_replica             : -1[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - log_on_each_node              : True[0m
+[35m[2025-11-01 14:28:47,806] [   DEBUG][0m - logging_dir                   : tmp_train_mmarco.jsonl/LLARA-passage-paddle/runs/Nov01_14-28-20_b63e119a1648[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - logging_first_step            : False[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - logging_steps                 : 1[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - logging_strategy              : IntervalStrategy.STEPS[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - logical_process_index         : 0[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - lr_end                        : 1e-07[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - lr_scheduler_type             : SchedulerType.LINEAR[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - margin                        : 0.2[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - matryoshka_dims               : [64, 128, 256, 512, 768][0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - matryoshka_loss_weights       : [1, 1, 1, 1, 1][0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - max_evaluate_steps            : -1[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - max_grad_norm                 : 1.0[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - max_steps                     : 100[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - metric_for_best_model         : None[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - metrics_output_path           : None[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - min_lr                        : 0.0[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - minimum_eval_times            : None[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - moe_sharding_parallel_degree  : 1[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - nccl_comm_group_config        : None[0m
+[35m[2025-11-01 14:28:47,807] [   DEBUG][0m - negatives_cross_device        : True[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - no_cuda                       : False[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - num_cycles                    : 0.5[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - num_train_epochs              : 1.0[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - offload_optim                 : False[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - optim                         : OptimizerNames.ADAMW[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - optimizer_name_suffix         : shard00[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - ordered_save_group_size       : 0[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - output_dir                    : tmp_train_mmarco.jsonl/LLARA-passage-paddle[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - output_signal_dir             : tmp_train_mmarco.jsonl/LLARA-passage-paddle[0m
+[35m[2025-11-01 14:28:47,808] [   DEBUG][0m - overwrite_output_dir          : True[0m
+[35m[2025-11-01 14:28:47,810] [   DEBUG][0m - pad_token_id                  : 0[0m
+[35m[2025-11-01 14:28:47,810] [   DEBUG][0m - past_index                    : -1[0m
+[35m[2025-11-01 14:28:47,810] [   DEBUG][0m - pdc_download_ckpt             : False[0m
+[35m[2025-11-01 14:28:47,813] [   DEBUG][0m - pdc_download_timeout          : 300[0m
+[35m[2025-11-01 14:28:47,815] [   DEBUG][0m - per_device_eval_batch_size    : 8[0m
+[35m[2025-11-01 14:28:47,815] [   DEBUG][0m - per_device_train_batch_size   : 1[0m
+[35m[2025-11-01 14:28:47,815] [   DEBUG][0m - pipeline_parallel_config      : [0m
+[35m[2025-11-01 14:28:47,818] [   DEBUG][0m - pipeline_parallel_degree      : 1[0m
+[35m[2025-11-01 14:28:47,818] [   DEBUG][0m - pipeline_parallel_rank        : 0[0m
+[35m[2025-11-01 14:28:47,818] [   DEBUG][0m - power                         : 1.0[0m
+[35m[2025-11-01 14:28:47,842] [   DEBUG][0m - prediction_loss_only          : False[0m
+[35m[2025-11-01 14:28:47,844] [   DEBUG][0m - process_index                 : 0[0m
+[35m[2025-11-01 14:28:47,844] [   DEBUG][0m - recompute                     : True[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - refined_recompute             : {}[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - release_grads                 : False[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - remove_unused_columns         : True[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - report_to                     : ['visualdl'][0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - resume_from_checkpoint        : None[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - run_name                      : tmp_train_mmarco.jsonl/LLARA-passage-paddle[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - save_checkpoint_format        : None[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - save_on_each_node             : False[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - save_rng_states               : True[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - save_sharded_model            : False[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - save_sharding_stage1_model_include_freeze_params: False[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - save_steps                    : 999999999[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - save_strategy                 : IntervalStrategy.STEPS[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - save_tokenizer                : True[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - save_total_limit              : None[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - scale_loss                    : 32768[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - seed                          : 42[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - sentence_pooling_method       : last_8[0m
+[35m[2025-11-01 14:28:47,859] [   DEBUG][0m - sep_parallel_degree           : 1[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sequence_parallel             : False[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sequence_parallel_config      : [0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sharded_model_from_ema        : False[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sharding                      : [<ShardingOption.FULL_SHARD: 'stage3'>, <ShardingOption.OFFLOAD: 'offload'>][0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sharding_comm_buffer_size_MB  : -1[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sharding_degree               : -1[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sharding_offload_opt_buffersize_GB: -1[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sharding_parallel_config      : [0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sharding_parallel_degree      : 4[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sharding_parallel_mesh_dimension: dp[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - sharding_parallel_rank        : 0[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - should_load_dataset           : True[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - should_load_sharding_stage1_model: False[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - should_log                    : True[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - should_save                   : True[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - should_save_model_state       : True[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - should_save_model_with_tensor_fusion: False[0m
+[35m[2025-11-01 14:28:47,860] [   DEBUG][0m - should_save_sharding_stage1_model: False[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - skip_data_intervals           : None[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - skip_memory_metrics           : True[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - skip_profile_timer            : True[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - split_inputs_sequence_dim     : True[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - split_norm_comm               : False[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - temperature                   : 0.01[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - tensor_parallel_config        : [0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - tensor_parallel_degree        : 1[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - tensor_parallel_rank          : 0[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - tensorwise_offload_optimizer  : False[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - to_static                     : False[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - train_batch_size              : 1[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - unified_checkpoint            : False[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - unified_checkpoint_config     : [0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - use_async_save                : False[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - use_expert_parallel           : False[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - use_hybrid_parallel           : True[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - use_inbatch_neg               : False[0m
+[35m[2025-11-01 14:28:47,861] [   DEBUG][0m - use_lowprecision_moment       : False[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - use_matryoshka                : False[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - wandb_api_key                 : None[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - wandb_http_proxy              : None[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - warmup_ratio                  : 0.0[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - warmup_steps                  : 10[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - weight_decay                  : 0.0[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - weight_name_suffix            : [0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - world_size                    : 4[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - zcc_ema_interval              : 1[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - zcc_ema_loss_threshold        : None[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - zcc_pipeline_hooks_capacity_usage: 0.6[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - zcc_save_ema_coef             : None[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - zcc_workers_num               : 3[0m
+[35m[2025-11-01 14:28:47,862] [   DEBUG][0m - [0m
+[32m[2025-11-01 14:28:47,862] [    INFO][0m - Starting training from resume_from_checkpoint : None[0m
+W1101 14:28:48.050137 60039 nccl_comm_context.cc:70] ncclCommInitRankConfigMemOpt is not supported.
+[2025-11-01 14:28:48,655] [ WARNING] group_sharded.py:148 - the input of scaler is None, please ensure the logic of your scaler outside is same as GroupShardedScaler.
+[2025-11-01 14:28:48,657] [ WARNING] group_sharded_stage3.py:195 - While using ClipGradByGlobalNorm in GroupShardedStage3, the grad clip of original optimizer will be changed.
+W1101 14:28:48.660228 60039 nccl_comm_context.cc:70] ncclCommInitRankConfigMemOpt is not supported.
+[32m[2025-11-01 14:29:22,100] [    INFO][0m - [timelog] checkpoint loading time: 0.00s (2025-11-01 14:29:22) [0m
+[32m[2025-11-01 14:29:22,100] [    INFO][0m - ***** Running training *****[0m
+[32m[2025-11-01 14:29:22,100] [    INFO][0m -   Num examples = 200,001[0m
+[32m[2025-11-01 14:29:22,100] [    INFO][0m -   Num Epochs = 1[0m
+[32m[2025-11-01 14:29:22,100] [    INFO][0m -   Instantaneous batch size per device = 1[0m
+[32m[2025-11-01 14:29:22,100] [    INFO][0m -   Total train batch size (w. parallel, distributed & accumulation) = 128[0m
+[32m[2025-11-01 14:29:22,100] [    INFO][0m -   Gradient Accumulation steps = 32[0m
+[32m[2025-11-01 14:29:22,100] [    INFO][0m -   Total optimization steps = 100[0m
+[32m[2025-11-01 14:29:22,100] [    INFO][0m -   Total num train samples = 12,800[0m
+[35m[2025-11-01 14:29:22,105] [   DEBUG][0m -   Number of trainable parameters = 79,953,920 (per device)[0m
+TrainProcess:   0%|          | 0/100 [00:00<?, ?it/s]W1101 14:29:22.466625 60039 multiply_fwd_func.cc:82] got different data type, run type promotion automatically, this may cause data type been changed.
+W1101 14:29:48.166923 60039 dygraph_functions.cc:94044] got different data type, run type promotion automatically, this may cause data type been changed.
+Found inf or nan, current scale is: 32768.0, decrease to: 32768.0*0.5
+[33m[2025-11-01 14:35:07,620] [ WARNING][0m - optimizer not run, scale_before: 32768.0, scale_after: 16384.0[0m
+TrainProcess:   1%|          | 1/100 [05:45<9:30:47, 345.94s/it][32m[2025-11-01 14:35:08,192] [    INFO][0m - loss: 5.30142212, learning_rate: 0.0, global_step: 1, current_memory_allocated: 12.27591347694397, current_memory_reserved: 70.46749949455261, max_memory_allocated: 57.78868627548218, max_memory_reserved: 70.46749949455261, interval_runtime: 345.9863, interval_samples_per_second: 0.37, interval_steps_per_second: 0.0029, progress_or_epoch: 0.0006, cpu_used_memory: 58.84, cpu_available_memory: 942.01[0m
+TrainProcess:   2%|▏         | 2/100 [12:21<10:12:44, 375.15s/it][32m[2025-11-01 14:41:43,774] [    INFO][0m - loss: 4.95529175, learning_rate: 1e-05, global_step: 2, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 395.6034, interval_samples_per_second: 0.3236, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0013, cpu_used_memory: 59.13, cpu_available_memory: 941.72[0m
+TrainProcess:   3%|▎         | 3/100 [18:57<10:21:40, 384.54s/it][32m[2025-11-01 14:48:19,528] [    INFO][0m - loss: 4.40704346, learning_rate: 2e-05, global_step: 3, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 395.7212, interval_samples_per_second: 0.3235, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0019, cpu_used_memory: 58.84, cpu_available_memory: 942.0[0m
+TrainProcess:   4%|▍         | 4/100 [25:36<10:24:42, 390.45s/it][32m[2025-11-01 14:54:58,994] [    INFO][0m - loss: 4.34732056, learning_rate: 3e-05, global_step: 4, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 399.5006, interval_samples_per_second: 0.3204, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0026, cpu_used_memory: 58.97, cpu_available_memory: 941.88[0m
+TrainProcess:   5%|▌         | 5/100 [32:17<10:23:59, 394.10s/it][32m[2025-11-01 15:01:39,604] [    INFO][0m - loss: 3.96940231, learning_rate: 4e-05, global_step: 5, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.5922, interval_samples_per_second: 0.3195, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0032, cpu_used_memory: 58.87, cpu_available_memory: 941.97[0m
+TrainProcess:   6%|▌         | 6/100 [38:57<10:20:48, 396.26s/it][32m[2025-11-01 15:08:20,005] [    INFO][0m - loss: 5.03268433, learning_rate: 5e-05, global_step: 6, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.4414, interval_samples_per_second: 0.3196, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0038, cpu_used_memory: 58.83, cpu_available_memory: 942.01[0m
+TrainProcess:   7%|▋         | 7/100 [45:40<10:17:27, 398.36s/it][32m[2025-11-01 15:15:02,713] [    INFO][0m - loss: 4.47592163, learning_rate: 6e-05, global_step: 7, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.682, interval_samples_per_second: 0.3179, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0045, cpu_used_memory: 59.24, cpu_available_memory: 941.6[0m
+TrainProcess:   8%|▊         | 8/100 [52:21<10:12:13, 399.28s/it][32m[2025-11-01 15:21:43,976] [    INFO][0m - loss: 3.82901001, learning_rate: 7e-05, global_step: 8, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.2334, interval_samples_per_second: 0.319, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0051, cpu_used_memory: 58.89, cpu_available_memory: 941.96[0m
+TrainProcess:   9%|▉         | 9/100 [59:03<10:06:54, 400.16s/it][32m[2025-11-01 15:28:26,104] [    INFO][0m - loss: 3.71914101, learning_rate: 8e-05, global_step: 9, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.0931, interval_samples_per_second: 0.3183, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0058, cpu_used_memory: 59.1, cpu_available_memory: 941.74[0m
+TrainProcess:  10%|█         | 10/100 [1:05:44<10:00:37, 400.41s/it][32m[2025-11-01 15:35:07,017] [    INFO][0m - loss: 2.48863506, learning_rate: 9e-05, global_step: 10, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.9907, interval_samples_per_second: 0.3192, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0064, cpu_used_memory: 58.98, cpu_available_memory: 941.86[0m
+TrainProcess:  11%|█         | 11/100 [1:12:24<9:53:49, 400.33s/it] [32m[2025-11-01 15:41:47,159] [    INFO][0m - loss: 2.21033478, learning_rate: 0.0001, global_step: 11, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.1382, interval_samples_per_second: 0.3199, interval_steps_per_second: 0.0025, progress_or_epoch: 0.007, cpu_used_memory: 59.08, cpu_available_memory: 941.77[0m
+TrainProcess:  12%|█▏        | 12/100 [1:19:04<9:46:39, 400.00s/it][32m[2025-11-01 15:48:26,472] [    INFO][0m - loss: 1.8239193, learning_rate: 9.889e-05, global_step: 12, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 399.2316, interval_samples_per_second: 0.3206, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0077, cpu_used_memory: 59.37, cpu_available_memory: 941.47[0m
+TrainProcess:  13%|█▎        | 13/100 [1:25:44<9:40:06, 400.08s/it][32m[2025-11-01 15:55:06,708] [    INFO][0m - loss: 1.05939865, learning_rate: 9.778e-05, global_step: 13, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.2797, interval_samples_per_second: 0.3198, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0083, cpu_used_memory: 59.27, cpu_available_memory: 941.57[0m
+TrainProcess:  14%|█▍        | 14/100 [1:32:24<9:33:25, 400.07s/it][32m[2025-11-01 16:01:46,794] [    INFO][0m - loss: 0.69354963, learning_rate: 9.667e-05, global_step: 14, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.0395, interval_samples_per_second: 0.32, interval_steps_per_second: 0.0025, progress_or_epoch: 0.009, cpu_used_memory: 59.29, cpu_available_memory: 941.56[0m
+TrainProcess:  15%|█▌        | 15/100 [1:39:07<9:28:00, 400.94s/it][32m[2025-11-01 16:08:29,712] [    INFO][0m - loss: 0.70521331, learning_rate: 9.556e-05, global_step: 15, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.9701, interval_samples_per_second: 0.3176, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0096, cpu_used_memory: 59.25, cpu_available_memory: 941.6[0m
+TrainProcess:  16%|█▌        | 16/100 [1:45:49<9:21:41, 401.21s/it][32m[2025-11-01 16:15:11,543] [    INFO][0m - loss: 0.67456877, learning_rate: 9.444e-05, global_step: 16, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.8229, interval_samples_per_second: 0.3185, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0102, cpu_used_memory: 59.25, cpu_available_memory: 941.6[0m
+TrainProcess:  17%|█▋        | 17/100 [1:52:30<9:15:02, 401.24s/it][32m[2025-11-01 16:21:52,906] [    INFO][0m - loss: 0.54681343, learning_rate: 9.333e-05, global_step: 17, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.2979, interval_samples_per_second: 0.319, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0109, cpu_used_memory: 59.59, cpu_available_memory: 941.25[0m
+TrainProcess:  18%|█▊        | 18/100 [1:59:11<9:08:18, 401.20s/it][32m[2025-11-01 16:28:33,944] [    INFO][0m - loss: 0.71270275, learning_rate: 9.222e-05, global_step: 18, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.1275, interval_samples_per_second: 0.3191, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0115, cpu_used_memory: 59.33, cpu_available_memory: 941.52[0m
+TrainProcess:  19%|█▉        | 19/100 [2:05:52<9:01:35, 401.18s/it][32m[2025-11-01 16:35:15,106] [    INFO][0m - loss: 0.66362476, learning_rate: 9.111e-05, global_step: 19, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.1678, interval_samples_per_second: 0.3191, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0122, cpu_used_memory: 59.51, cpu_available_memory: 941.34[0m
+TrainProcess:  20%|██        | 20/100 [2:12:34<8:54:57, 401.22s/it][32m[2025-11-01 16:41:56,434] [    INFO][0m - loss: 0.51348567, learning_rate: 9e-05, global_step: 20, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.283, interval_samples_per_second: 0.319, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0128, cpu_used_memory: 59.41, cpu_available_memory: 941.44[0m
+TrainProcess:  21%|██        | 21/100 [2:19:14<8:47:54, 400.95s/it][32m[2025-11-01 16:48:36,652] [    INFO][0m - loss: 0.49842453, learning_rate: 8.889e-05, global_step: 21, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.2941, interval_samples_per_second: 0.3198, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0134, cpu_used_memory: 59.41, cpu_available_memory: 941.43[0m
+TrainProcess:  22%|██▏       | 22/100 [2:25:55<8:41:22, 401.06s/it][32m[2025-11-01 16:55:18,030] [    INFO][0m - loss: 0.39264494, learning_rate: 8.778e-05, global_step: 22, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.3313, interval_samples_per_second: 0.3189, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0141, cpu_used_memory: 59.86, cpu_available_memory: 940.99[0m
+TrainProcess:  23%|██▎       | 23/100 [2:32:37<8:35:00, 401.30s/it][32m[2025-11-01 17:01:59,888] [    INFO][0m - loss: 0.46471953, learning_rate: 8.667e-05, global_step: 23, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.8532, interval_samples_per_second: 0.3185, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0147, cpu_used_memory: 59.52, cpu_available_memory: 941.33[0m
+TrainProcess:  24%|██▍       | 24/100 [2:39:19<8:28:35, 401.51s/it][32m[2025-11-01 17:08:41,953] [    INFO][0m - loss: 0.47722584, learning_rate: 8.556e-05, global_step: 24, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.0355, interval_samples_per_second: 0.3184, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0154, cpu_used_memory: 59.77, cpu_available_memory: 941.08[0m
+TrainProcess:  25%|██▌       | 25/100 [2:46:02<8:22:31, 402.02s/it][32m[2025-11-01 17:15:25,118] [    INFO][0m - loss: 0.42750406, learning_rate: 8.444e-05, global_step: 25, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.2014, interval_samples_per_second: 0.3175, interval_steps_per_second: 0.0025, progress_or_epoch: 0.016, cpu_used_memory: 59.63, cpu_available_memory: 941.22[0m
+TrainProcess:  26%|██▌       | 26/100 [2:52:44<8:15:37, 401.86s/it][32m[2025-11-01 17:22:06,565] [    INFO][0m - loss: 0.37504244, learning_rate: 8.333e-05, global_step: 26, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.4871, interval_samples_per_second: 0.3188, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0166, cpu_used_memory: 59.66, cpu_available_memory: 941.18[0m
+TrainProcess:  27%|██▋       | 27/100 [2:59:26<8:09:06, 402.01s/it][32m[2025-11-01 17:28:48,893] [    INFO][0m - loss: 0.29181445, learning_rate: 8.222e-05, global_step: 27, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.329, interval_samples_per_second: 0.3181, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0173, cpu_used_memory: 59.64, cpu_available_memory: 941.2[0m
+TrainProcess:  28%|██▊       | 28/100 [3:06:07<8:01:55, 401.60s/it][32m[2025-11-01 17:35:29,576] [    INFO][0m - loss: 0.45821714, learning_rate: 8.111e-05, global_step: 28, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.6679, interval_samples_per_second: 0.3195, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0179, cpu_used_memory: 59.68, cpu_available_memory: 941.16[0m
+TrainProcess:  29%|██▉       | 29/100 [3:12:51<7:56:03, 402.30s/it][32m[2025-11-01 17:42:13,537] [    INFO][0m - loss: 0.32785797, learning_rate: 8e-05, global_step: 29, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.9482, interval_samples_per_second: 0.3169, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0186, cpu_used_memory: 60.07, cpu_available_memory: 940.77[0m
+TrainProcess:  30%|███       | 30/100 [3:19:33<7:49:28, 402.41s/it][32m[2025-11-01 17:48:56,176] [    INFO][0m - loss: 0.61253774, learning_rate: 7.889e-05, global_step: 30, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.629, interval_samples_per_second: 0.3179, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0192, cpu_used_memory: 59.84, cpu_available_memory: 941.01[0m
+TrainProcess:  31%|███       | 31/100 [3:26:15<7:42:29, 402.16s/it][32m[2025-11-01 17:55:37,809] [    INFO][0m - loss: 0.27006173, learning_rate: 7.778e-05, global_step: 31, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.6024, interval_samples_per_second: 0.3187, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0198, cpu_used_memory: 60.01, cpu_available_memory: 940.83[0m
+TrainProcess:  32%|███▏      | 32/100 [3:32:57<7:35:34, 401.98s/it][32m[2025-11-01 18:02:19,305] [    INFO][0m - loss: 0.2481395, learning_rate: 7.667e-05, global_step: 32, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.5499, interval_samples_per_second: 0.3188, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0205, cpu_used_memory: 59.9, cpu_available_memory: 940.94[0m
+TrainProcess:  33%|███▎      | 33/100 [3:39:38<7:28:39, 401.79s/it][32m[2025-11-01 18:09:00,719] [    INFO][0m - loss: 0.39536506, learning_rate: 7.556e-05, global_step: 33, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.344, interval_samples_per_second: 0.3189, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0211, cpu_used_memory: 59.9, cpu_available_memory: 940.95[0m
+TrainProcess:  34%|███▍      | 34/100 [3:46:19<7:21:47, 401.62s/it][32m[2025-11-01 18:15:41,896] [    INFO][0m - loss: 0.38513905, learning_rate: 7.444e-05, global_step: 34, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.2271, interval_samples_per_second: 0.319, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0218, cpu_used_memory: 59.95, cpu_available_memory: 940.9[0m
+TrainProcess:  35%|███▌      | 35/100 [3:53:01<7:15:08, 401.67s/it][32m[2025-11-01 18:22:23,673] [    INFO][0m - loss: 0.44294834, learning_rate: 7.333e-05, global_step: 35, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.7659, interval_samples_per_second: 0.3186, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0224, cpu_used_memory: 59.95, cpu_available_memory: 940.89[0m
+TrainProcess:  36%|███▌      | 36/100 [3:59:42<7:08:09, 401.39s/it][32m[2025-11-01 18:29:04,434] [    INFO][0m - loss: 0.26775789, learning_rate: 7.222e-05, global_step: 36, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.7623, interval_samples_per_second: 0.3194, interval_steps_per_second: 0.0025, progress_or_epoch: 0.023, cpu_used_memory: 60.31, cpu_available_memory: 940.53[0m
+TrainProcess:  37%|███▋      | 37/100 [4:06:22<7:01:16, 401.21s/it][32m[2025-11-01 18:35:45,283] [    INFO][0m - loss: 0.43162769, learning_rate: 7.111e-05, global_step: 37, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.8296, interval_samples_per_second: 0.3193, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0237, cpu_used_memory: 60.01, cpu_available_memory: 940.84[0m
+TrainProcess:  38%|███▊      | 38/100 [4:13:04<6:54:39, 401.29s/it][32m[2025-11-01 18:42:26,705] [    INFO][0m - loss: 0.28665954, learning_rate: 7e-05, global_step: 38, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.4367, interval_samples_per_second: 0.3189, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0243, cpu_used_memory: 60.31, cpu_available_memory: 940.54[0m
+TrainProcess:  39%|███▉      | 39/100 [4:19:46<6:48:12, 401.51s/it][32m[2025-11-01 18:49:08,707] [    INFO][0m - loss: 0.41991347, learning_rate: 6.889e-05, global_step: 39, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.0234, interval_samples_per_second: 0.3184, interval_steps_per_second: 0.0025, progress_or_epoch: 0.025, cpu_used_memory: 60.15, cpu_available_memory: 940.7[0m
+TrainProcess:  40%|████      | 40/100 [4:26:28<6:41:34, 401.57s/it][32m[2025-11-01 18:55:50,428] [    INFO][0m - loss: 0.35437369, learning_rate: 6.778e-05, global_step: 40, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.7024, interval_samples_per_second: 0.3186, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0256, cpu_used_memory: 60.2, cpu_available_memory: 940.65[0m
+TrainProcess:  41%|████      | 41/100 [4:33:10<6:34:58, 401.67s/it][32m[2025-11-01 19:02:32,306] [    INFO][0m - loss: 0.33553171, learning_rate: 6.667e-05, global_step: 41, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.8947, interval_samples_per_second: 0.3185, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0262, cpu_used_memory: 60.2, cpu_available_memory: 940.65[0m
+TrainProcess:  42%|████▏     | 42/100 [4:39:50<6:27:52, 401.25s/it][32m[2025-11-01 19:09:12,611] [    INFO][0m - loss: 0.40186411, learning_rate: 6.556e-05, global_step: 42, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.2608, interval_samples_per_second: 0.3198, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0269, cpu_used_memory: 60.27, cpu_available_memory: 940.57[0m
+TrainProcess:  43%|████▎     | 43/100 [4:46:32<6:21:23, 401.46s/it][32m[2025-11-01 19:15:54,551] [    INFO][0m - loss: 0.13329619, learning_rate: 6.444e-05, global_step: 43, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.0102, interval_samples_per_second: 0.3184, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0275, cpu_used_memory: 60.68, cpu_available_memory: 940.17[0m
+TrainProcess:  44%|████▍     | 44/100 [4:53:13<6:14:29, 401.24s/it][32m[2025-11-01 19:22:35,350] [    INFO][0m - loss: 0.34458822, learning_rate: 6.333e-05, global_step: 44, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.7199, interval_samples_per_second: 0.3194, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0282, cpu_used_memory: 60.28, cpu_available_memory: 940.56[0m
+TrainProcess:  45%|████▌     | 45/100 [4:59:53<6:07:37, 401.04s/it][32m[2025-11-01 19:29:15,867] [    INFO][0m - loss: 0.37106729, learning_rate: 6.222e-05, global_step: 45, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.552, interval_samples_per_second: 0.3196, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0288, cpu_used_memory: 60.61, cpu_available_memory: 940.23[0m
+TrainProcess:  46%|████▌     | 46/100 [5:06:35<6:01:08, 401.27s/it][32m[2025-11-01 19:35:57,659] [    INFO][0m - loss: 0.27576214, learning_rate: 6.111e-05, global_step: 46, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.7828, interval_samples_per_second: 0.3186, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0294, cpu_used_memory: 60.3, cpu_available_memory: 940.55[0m
+TrainProcess:  47%|████▋     | 47/100 [5:13:17<5:54:34, 401.41s/it][32m[2025-11-01 19:42:39,385] [    INFO][0m - loss: 0.13976067, learning_rate: 6e-05, global_step: 47, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.7574, interval_samples_per_second: 0.3186, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0301, cpu_used_memory: 60.53, cpu_available_memory: 940.32[0m
+TrainProcess:  48%|████▊     | 48/100 [5:19:58<5:47:53, 401.41s/it][32m[2025-11-01 19:49:20,848] [    INFO][0m - loss: 0.23178703, learning_rate: 5.889e-05, global_step: 48, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.415, interval_samples_per_second: 0.3189, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0307, cpu_used_memory: 60.38, cpu_available_memory: 940.47[0m
+TrainProcess:  49%|████▉     | 49/100 [5:26:39<5:41:09, 401.37s/it][32m[2025-11-01 19:56:02,051] [    INFO][0m - loss: 0.3006193, learning_rate: 5.778e-05, global_step: 49, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.2432, interval_samples_per_second: 0.319, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0314, cpu_used_memory: 60.49, cpu_available_memory: 940.36[0m
+TrainProcess:  50%|█████     | 50/100 [5:33:22<5:34:54, 401.90s/it][32m[2025-11-01 20:02:45,234] [    INFO][0m - loss: 0.32214558, learning_rate: 5.667e-05, global_step: 50, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.1312, interval_samples_per_second: 0.3175, interval_steps_per_second: 0.0025, progress_or_epoch: 0.032, cpu_used_memory: 60.45, cpu_available_memory: 940.39[0m
+TrainProcess:  51%|█████     | 51/100 [5:40:04<5:28:01, 401.67s/it][32m[2025-11-01 20:09:26,353] [    INFO][0m - loss: 0.21646398, learning_rate: 5.556e-05, global_step: 51, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.1653, interval_samples_per_second: 0.3191, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0326, cpu_used_memory: 60.47, cpu_available_memory: 940.38[0m
+TrainProcess:  52%|█████▏    | 52/100 [5:46:44<5:21:05, 401.35s/it][32m[2025-11-01 20:16:06,930] [    INFO][0m - loss: 0.2939086, learning_rate: 5.444e-05, global_step: 52, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.6021, interval_samples_per_second: 0.3195, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0333, cpu_used_memory: 60.97, cpu_available_memory: 939.88[0m
+TrainProcess:  53%|█████▎    | 53/100 [5:53:27<5:14:40, 401.71s/it][32m[2025-11-01 20:22:49,519] [    INFO][0m - loss: 0.12507194, learning_rate: 5.333e-05, global_step: 53, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.5341, interval_samples_per_second: 0.318, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0339, cpu_used_memory: 60.58, cpu_available_memory: 940.27[0m
+TrainProcess:  54%|█████▍    | 54/100 [6:00:07<5:07:39, 401.30s/it][32m[2025-11-01 20:29:29,842] [    INFO][0m - loss: 0.20214003, learning_rate: 5.222e-05, global_step: 54, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.3319, interval_samples_per_second: 0.3197, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0346, cpu_used_memory: 60.93, cpu_available_memory: 939.91[0m
+TrainProcess:  55%|█████▌    | 55/100 [6:06:50<5:01:21, 401.82s/it][32m[2025-11-01 20:36:12,896] [    INFO][0m - loss: 0.22510302, learning_rate: 5.111e-05, global_step: 55, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.0657, interval_samples_per_second: 0.3176, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0352, cpu_used_memory: 60.61, cpu_available_memory: 940.23[0m
+TrainProcess:  56%|█████▌    | 56/100 [6:13:32<4:54:46, 401.96s/it][32m[2025-11-01 20:42:55,194] [    INFO][0m - loss: 0.63401538, learning_rate: 5e-05, global_step: 56, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.294, interval_samples_per_second: 0.3182, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0358, cpu_used_memory: 60.92, cpu_available_memory: 939.93[0m
+TrainProcess:  57%|█████▋    | 57/100 [6:20:16<4:48:18, 402.30s/it][32m[2025-11-01 20:49:38,264] [    INFO][0m - loss: 0.45384973, learning_rate: 4.889e-05, global_step: 57, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.085, interval_samples_per_second: 0.3176, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0365, cpu_used_memory: 60.74, cpu_available_memory: 940.1[0m
+TrainProcess:  58%|█████▊    | 58/100 [6:26:58<4:41:38, 402.35s/it][32m[2025-11-01 20:56:20,801] [    INFO][0m - loss: 0.20536405, learning_rate: 4.778e-05, global_step: 58, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.4519, interval_samples_per_second: 0.3181, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0371, cpu_used_memory: 60.93, cpu_available_memory: 939.92[0m
+TrainProcess:  59%|█████▉    | 59/100 [6:33:41<4:35:01, 402.48s/it][32m[2025-11-01 21:03:03,501] [    INFO][0m - loss: 0.18170655, learning_rate: 4.667e-05, global_step: 59, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.7849, interval_samples_per_second: 0.3178, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0378, cpu_used_memory: 60.78, cpu_available_memory: 940.06[0m
+TrainProcess:  60%|██████    | 60/100 [6:40:23<4:28:22, 402.56s/it][32m[2025-11-01 21:09:46,224] [    INFO][0m - loss: 0.24450195, learning_rate: 4.556e-05, global_step: 60, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.7386, interval_samples_per_second: 0.3178, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0384, cpu_used_memory: 60.89, cpu_available_memory: 939.96[0m
+TrainProcess:  61%|██████    | 61/100 [6:47:07<4:21:51, 402.87s/it][32m[2025-11-01 21:16:29,899] [    INFO][0m - loss: 0.19175458, learning_rate: 4.444e-05, global_step: 61, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.6306, interval_samples_per_second: 0.3171, interval_steps_per_second: 0.0025, progress_or_epoch: 0.039, cpu_used_memory: 60.85, cpu_available_memory: 939.99[0m
+TrainProcess:  62%|██████▏   | 62/100 [6:53:48<4:14:45, 402.25s/it][32m[2025-11-01 21:23:10,635] [    INFO][0m - loss: 0.21944112, learning_rate: 4.333e-05, global_step: 62, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.7743, interval_samples_per_second: 0.3194, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0397, cpu_used_memory: 60.86, cpu_available_memory: 939.98[0m
+TrainProcess:  63%|██████▎   | 63/100 [7:00:28<4:07:39, 401.60s/it][32m[2025-11-01 21:29:50,746] [    INFO][0m - loss: 0.14639384, learning_rate: 4.222e-05, global_step: 63, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.0911, interval_samples_per_second: 0.3199, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0403, cpu_used_memory: 61.36, cpu_available_memory: 939.49[0m
+TrainProcess:  64%|██████▍   | 64/100 [7:07:09<4:00:46, 401.29s/it][32m[2025-11-01 21:36:31,309] [    INFO][0m - loss: 0.23113734, learning_rate: 4.111e-05, global_step: 64, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.5491, interval_samples_per_second: 0.3196, interval_steps_per_second: 0.0025, progress_or_epoch: 0.041, cpu_used_memory: 61.02, cpu_available_memory: 939.83[0m
+TrainProcess:  65%|██████▌   | 65/100 [7:13:52<3:54:27, 401.92s/it][32m[2025-11-01 21:43:14,717] [    INFO][0m - loss: 0.23736763, learning_rate: 4e-05, global_step: 65, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.3774, interval_samples_per_second: 0.3173, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0416, cpu_used_memory: 61.38, cpu_available_memory: 939.47[0m
+TrainProcess:  66%|██████▌   | 66/100 [7:20:33<3:47:39, 401.74s/it][32m[2025-11-01 21:49:55,997] [    INFO][0m - loss: 0.34425539, learning_rate: 3.889e-05, global_step: 66, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.3186, interval_samples_per_second: 0.3189, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0422, cpu_used_memory: 61.04, cpu_available_memory: 939.81[0m
+TrainProcess:  67%|██████▋   | 67/100 [7:27:16<3:41:04, 401.95s/it][32m[2025-11-01 21:56:38,481] [    INFO][0m - loss: 0.44939297, learning_rate: 3.778e-05, global_step: 67, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.4617, interval_samples_per_second: 0.318, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0429, cpu_used_memory: 61.37, cpu_available_memory: 939.47[0m
+TrainProcess:  68%|██████▊   | 68/100 [7:33:58<3:34:22, 401.97s/it][32m[2025-11-01 22:03:20,441] [    INFO][0m - loss: 0.27526224, learning_rate: 3.667e-05, global_step: 68, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.0161, interval_samples_per_second: 0.3184, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0435, cpu_used_memory: 61.15, cpu_available_memory: 939.69[0m
+TrainProcess:  69%|██████▉   | 69/100 [7:40:41<3:27:53, 402.38s/it][32m[2025-11-01 22:10:03,808] [    INFO][0m - loss: 0.44756508, learning_rate: 3.556e-05, global_step: 69, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.3586, interval_samples_per_second: 0.3173, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0442, cpu_used_memory: 61.38, cpu_available_memory: 939.46[0m
+TrainProcess:  70%|███████   | 70/100 [7:47:25<3:21:26, 402.87s/it][32m[2025-11-01 22:16:47,820] [    INFO][0m - loss: 0.2250793, learning_rate: 3.444e-05, global_step: 70, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.9849, interval_samples_per_second: 0.3168, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0448, cpu_used_memory: 61.21, cpu_available_memory: 939.64[0m
+TrainProcess:  71%|███████   | 71/100 [7:54:06<3:14:23, 402.19s/it][32m[2025-11-01 22:23:28,410] [    INFO][0m - loss: 0.18877041, learning_rate: 3.333e-05, global_step: 71, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.605, interval_samples_per_second: 0.3195, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0454, cpu_used_memory: 61.41, cpu_available_memory: 939.44[0m
+TrainProcess:  72%|███████▏  | 72/100 [8:00:47<3:07:33, 401.93s/it][32m[2025-11-01 22:30:09,712] [    INFO][0m - loss: 0.34880316, learning_rate: 3.222e-05, global_step: 72, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.3124, interval_samples_per_second: 0.319, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0461, cpu_used_memory: 61.28, cpu_available_memory: 939.57[0m
+TrainProcess:  73%|███████▎  | 73/100 [8:07:28<3:00:44, 401.64s/it][32m[2025-11-01 22:36:50,743] [    INFO][0m - loss: 0.19271833, learning_rate: 3.111e-05, global_step: 73, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.9848, interval_samples_per_second: 0.3192, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0467, cpu_used_memory: 61.42, cpu_available_memory: 939.43[0m
+TrainProcess:  74%|███████▍  | 74/100 [8:14:10<2:54:02, 401.65s/it][32m[2025-11-01 22:43:32,325] [    INFO][0m - loss: 0.20546728, learning_rate: 3e-05, global_step: 74, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.6562, interval_samples_per_second: 0.3187, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0474, cpu_used_memory: 61.34, cpu_available_memory: 939.51[0m
+TrainProcess:  75%|███████▌  | 75/100 [8:20:54<2:47:43, 402.52s/it][32m[2025-11-01 22:50:16,935] [    INFO][0m - loss: 0.36092532, learning_rate: 2.889e-05, global_step: 75, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 404.5728, interval_samples_per_second: 0.3164, interval_steps_per_second: 0.0025, progress_or_epoch: 0.048, cpu_used_memory: 61.39, cpu_available_memory: 939.45[0m
+TrainProcess:  76%|███████▌  | 76/100 [8:27:36<2:40:58, 402.43s/it][32m[2025-11-01 22:56:59,171] [    INFO][0m - loss: 0.17070693, learning_rate: 2.778e-05, global_step: 76, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.2183, interval_samples_per_second: 0.3182, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0486, cpu_used_memory: 61.4, cpu_available_memory: 939.44[0m
+TrainProcess:  77%|███████▋  | 77/100 [8:34:17<2:34:06, 402.02s/it][32m[2025-11-01 23:03:40,222] [    INFO][0m - loss: 0.26325572, learning_rate: 2.667e-05, global_step: 77, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.058, interval_samples_per_second: 0.3192, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0493, cpu_used_memory: 61.45, cpu_available_memory: 939.4[0m
+TrainProcess:  78%|███████▊  | 78/100 [8:40:59<2:27:19, 401.79s/it][32m[2025-11-01 23:10:21,483] [    INFO][0m - loss: 0.19824362, learning_rate: 2.556e-05, global_step: 78, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.2364, interval_samples_per_second: 0.319, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0499, cpu_used_memory: 61.47, cpu_available_memory: 939.37[0m
+TrainProcess:  79%|███████▉  | 79/100 [8:47:42<2:20:46, 402.24s/it][32m[2025-11-01 23:17:04,742] [    INFO][0m - loss: 0.29262131, learning_rate: 2.444e-05, global_step: 79, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.3043, interval_samples_per_second: 0.3174, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0506, cpu_used_memory: 61.57, cpu_available_memory: 939.28[0m
+TrainProcess:  80%|████████  | 80/100 [8:54:28<2:14:24, 403.24s/it][32m[2025-11-01 23:23:50,386] [    INFO][0m - loss: 0.2408849, learning_rate: 2.333e-05, global_step: 80, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 405.5868, interval_samples_per_second: 0.3156, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0512, cpu_used_memory: 61.96, cpu_available_memory: 938.88[0m
+TrainProcess:  81%|████████  | 81/100 [9:01:11<2:07:40, 403.17s/it][32m[2025-11-01 23:30:33,412] [    INFO][0m - loss: 0.21224731, learning_rate: 2.222e-05, global_step: 81, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.0366, interval_samples_per_second: 0.3176, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0518, cpu_used_memory: 61.62, cpu_available_memory: 939.22[0m
+TrainProcess:  82%|████████▏ | 82/100 [9:07:53<2:00:50, 402.80s/it][32m[2025-11-01 23:37:15,336] [    INFO][0m - loss: 0.30385292, learning_rate: 2.111e-05, global_step: 82, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.8803, interval_samples_per_second: 0.3185, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0525, cpu_used_memory: 62.06, cpu_available_memory: 938.78[0m
+TrainProcess:  83%|████████▎ | 83/100 [9:14:36<1:54:10, 403.00s/it][32m[2025-11-01 23:43:58,733] [    INFO][0m - loss: 0.16555732, learning_rate: 2e-05, global_step: 83, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.4522, interval_samples_per_second: 0.3173, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0531, cpu_used_memory: 61.72, cpu_available_memory: 939.13[0m
+TrainProcess:  84%|████████▍ | 84/100 [9:21:19<1:47:26, 402.92s/it][32m[2025-11-01 23:50:41,490] [    INFO][0m - loss: 0.20000756, learning_rate: 1.889e-05, global_step: 84, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.7624, interval_samples_per_second: 0.3178, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0538, cpu_used_memory: 62.07, cpu_available_memory: 938.77[0m
+TrainProcess:  85%|████████▌ | 85/100 [9:28:01<1:40:40, 402.73s/it][32m[2025-11-01 23:57:23,746] [    INFO][0m - loss: 0.25972348, learning_rate: 1.778e-05, global_step: 85, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.2673, interval_samples_per_second: 0.3182, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0544, cpu_used_memory: 61.79, cpu_available_memory: 939.06[0m
+TrainProcess:  86%|████████▌ | 86/100 [9:34:45<1:34:05, 403.25s/it][32m[2025-11-02 00:04:08,211] [    INFO][0m - loss: 0.20755601, learning_rate: 1.667e-05, global_step: 86, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 404.4627, interval_samples_per_second: 0.3165, interval_steps_per_second: 0.0025, progress_or_epoch: 0.055, cpu_used_memory: 62.13, cpu_available_memory: 938.72[0m
+TrainProcess:  87%|████████▋ | 87/100 [9:41:26<1:27:13, 402.57s/it][32m[2025-11-02 00:10:49,198] [    INFO][0m - loss: 0.29394907, learning_rate: 1.556e-05, global_step: 87, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.0134, interval_samples_per_second: 0.3192, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0557, cpu_used_memory: 61.8, cpu_available_memory: 939.04[0m
+TrainProcess:  88%|████████▊ | 88/100 [9:48:06<1:20:19, 401.64s/it][32m[2025-11-02 00:17:28,661] [    INFO][0m - loss: 0.31440371, learning_rate: 1.444e-05, global_step: 88, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 399.4526, interval_samples_per_second: 0.3204, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0563, cpu_used_memory: 62.11, cpu_available_memory: 938.74[0m
+TrainProcess:  89%|████████▉ | 89/100 [9:54:48<1:13:39, 401.77s/it][32m[2025-11-02 00:24:10,763] [    INFO][0m - loss: 0.26902431, learning_rate: 1.333e-05, global_step: 89, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.0705, interval_samples_per_second: 0.3184, interval_steps_per_second: 0.0025, progress_or_epoch: 0.057, cpu_used_memory: 61.91, cpu_available_memory: 938.93[0m
+TrainProcess:  90%|█████████ | 90/100 [10:01:30<1:06:58, 401.86s/it][32m[2025-11-02 00:30:52,780] [    INFO][0m - loss: 0.25527662, learning_rate: 1.222e-05, global_step: 90, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.0541, interval_samples_per_second: 0.3184, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0576, cpu_used_memory: 62.15, cpu_available_memory: 938.7[0m
+TrainProcess:  91%|█████████ | 91/100 [10:08:11<1:00:15, 401.73s/it][32m[2025-11-02 00:37:34,275] [    INFO][0m - loss: 0.341277, learning_rate: 1.111e-05, global_step: 91, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.4474, interval_samples_per_second: 0.3188, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0582, cpu_used_memory: 61.96, cpu_available_memory: 938.89[0m
+TrainProcess:  92%|█████████▏| 92/100 [10:14:57<53:42, 402.78s/it]  [32m[2025-11-02 00:44:19,448] [    INFO][0m - loss: 0.22811371, learning_rate: 1e-05, global_step: 92, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 405.2212, interval_samples_per_second: 0.3159, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0589, cpu_used_memory: 62.2, cpu_available_memory: 938.65[0m
+TrainProcess:  93%|█████████▎| 93/100 [10:21:39<46:58, 402.71s/it][32m[2025-11-02 00:51:01,970] [    INFO][0m - loss: 0.26922929, learning_rate: 8.889e-06, global_step: 93, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.5328, interval_samples_per_second: 0.318, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0595, cpu_used_memory: 61.98, cpu_available_memory: 938.86[0m
+TrainProcess:  94%|█████████▍| 94/100 [10:28:20<40:13, 402.18s/it][32m[2025-11-02 00:57:43,024] [    INFO][0m - loss: 0.32746762, learning_rate: 7.778e-06, global_step: 94, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 400.9806, interval_samples_per_second: 0.3192, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0602, cpu_used_memory: 62.21, cpu_available_memory: 938.64[0m
+TrainProcess:  95%|█████████▌| 95/100 [10:35:03<33:32, 402.49s/it][32m[2025-11-02 01:04:26,143] [    INFO][0m - loss: 0.30963302, learning_rate: 6.667e-06, global_step: 95, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.2043, interval_samples_per_second: 0.3175, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0608, cpu_used_memory: 62.07, cpu_available_memory: 938.77[0m
+TrainProcess:  96%|█████████▌| 96/100 [10:41:47<26:50, 402.69s/it][32m[2025-11-02 01:11:09,375] [    INFO][0m - loss: 0.28360158, learning_rate: 5.556e-06, global_step: 96, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 403.1701, interval_samples_per_second: 0.3175, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0614, cpu_used_memory: 62.3, cpu_available_memory: 938.54[0m
+TrainProcess:  97%|█████████▋| 97/100 [10:48:29<20:07, 402.49s/it][32m[2025-11-02 01:17:51,347] [    INFO][0m - loss: 0.24288189, learning_rate: 4.444e-06, global_step: 97, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.0027, interval_samples_per_second: 0.3184, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0621, cpu_used_memory: 62.17, cpu_available_memory: 938.67[0m
+TrainProcess:  98%|█████████▊| 98/100 [10:55:13<13:26, 403.11s/it][32m[2025-11-02 01:24:35,912] [    INFO][0m - loss: 0.40710437, learning_rate: 3.333e-06, global_step: 98, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 404.5509, interval_samples_per_second: 0.3164, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0627, cpu_used_memory: 62.35, cpu_available_memory: 938.5[0m
+TrainProcess:  99%|█████████▉| 99/100 [11:01:54<06:42, 402.56s/it][32m[2025-11-02 01:31:17,169] [    INFO][0m - loss: 0.2575587, learning_rate: 2.222e-06, global_step: 99, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 401.2805, interval_samples_per_second: 0.319, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0634, cpu_used_memory: 62.22, cpu_available_memory: 938.62[0m
+TrainProcess: 100%|██████████| 100/100 [11:08:37<00:00, 402.65s/it][32m[2025-11-02 01:38:00,063] [    INFO][0m - loss: 0.25714511, learning_rate: 1.111e-06, global_step: 100, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.46749949455261, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.46749949455261, interval_runtime: 402.8578, interval_samples_per_second: 0.3177, interval_steps_per_second: 0.0025, progress_or_epoch: 0.064, cpu_used_memory: 62.38, cpu_available_memory: 938.47[0m
+[32m[2025-11-02 01:38:00,063] [    INFO][0m -
+Training completed.
+[0m
+[32m[2025-11-02 01:38:00,095] [    INFO][0m - train_runtime: 40117.9587, train_samples_per_second: 0.3191, train_steps_per_second: 0.0025, train_loss: 0.7623166245222092, progress_or_epoch: 0.064, cpu_used_memory: 62.38, cpu_available_memory: 938.47[0m
+TrainProcess: 100%|██████████| 100/100 [11:08:37<00:00, 401.18s/it]
+[32m[2025-11-02 01:38:00,100] [    INFO][0m - Saving model checkpoint to tmp_train_mmarco.jsonl/LLARA-passage-paddle[0m
+[32m[2025-11-02 01:38:00,126] [    INFO][0m - tokenizer config file saved in tmp_train_mmarco.jsonl/LLARA-passage-paddle/tokenizer_config.json[0m
+[32m[2025-11-02 01:38:00,132] [    INFO][0m - Special tokens file saved in tmp_train_mmarco.jsonl/LLARA-passage-paddle/special_tokens_map.json[0m
+[32m[2025-11-02 01:38:00,172] [    INFO][0m - added tokens file saved in tmp_train_mmarco.jsonl/LLARA-passage-paddle/added_tokens.json[0m
+[32m[2025-11-02 01:38:02,660] [    INFO][0m - Configuration saved in tmp_train_mmarco.jsonl/LLARA-passage-paddle/config.json[0m
+[32m[2025-11-02 01:38:02,660] [    INFO][0m - ***** train metrics *****[0m
+[32m[2025-11-02 01:38:02,660] [    INFO][0m -   cpu_available_memory     =      938.47[0m
+[32m[2025-11-02 01:38:02,661] [    INFO][0m -   cpu_used_memory          =       62.38[0m
+[32m[2025-11-02 01:38:02,661] [    INFO][0m -   progress_or_epoch        =       0.064[0m
+[32m[2025-11-02 01:38:02,661] [    INFO][0m -   train_loss               =      0.7623[0m
+[32m[2025-11-02 01:38:02,661] [    INFO][0m -   train_runtime            = 11:08:37.95[0m
+[32m[2025-11-02 01:38:02,661] [    INFO][0m -   train_samples_per_second =      0.3191[0m
+[32m[2025-11-02 01:38:02,661] [    INFO][0m -   train_steps_per_second   =      0.0025[0m
+LAUNCH INFO 2025-11-02 01:38:11,625 Pod completed
+LAUNCH INFO 2025-11-02 01:38:11,626 Exit code 0

LLARA-passage-paddle/train_results.json ADDED Viewed

	@@ -0,0 +1,9 @@

+{
+    "cpu_available_memory": 938.47,
+    "cpu_used_memory": 62.38,
+    "progress_or_epoch": 0.064,
+    "train_loss": 0.7623166245222092,
+    "train_runtime": 40117.9587,
+    "train_samples_per_second": 0.3191,
+    "train_steps_per_second": 0.0025
+}

LLARA-passage-paddle/trainer_state.json ADDED Viewed

	@@ -0,0 +1,15 @@

+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "consumed_samples": 12800,
+  "epoch": 0.06399872002559949,
+  "global_step": 100,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [],
+  "max_steps": 100,
+  "num_train_epochs": 1,
+  "total_flos": 0,
+  "trial_name": null,
+  "trial_params": null
+}

LLARA-passage-paddle/training_args.bin ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:4322edc4d83209bdb0f4d77fd161c109654ccb80c138c1182014d7c9cc0accc0
+size 4030