Upload 12 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/lora_model_state.pdparams filter=lfs diff=lfs merge=lfs -text

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/all_results.json

@@ -0,0 +1,7 @@
+{
+    "progress_or_epoch": 0.064,
+    "train_loss": 0.1801042765378952,
+    "train_runtime": 40654.6818,
+    "train_samples_per_second": 0.3148,
+    "train_steps_per_second": 0.0025
+}

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/config.json

@@ -0,0 +1,41 @@
+{
+  "_name_or_path": "/ceph_home/lizhuoqun2021/hf_models/llama-2-7b",
+  "alibi": false,
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "dpo_config": null,
+  "dtype": "float16",
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "immediate_clear_past_key_value": false,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "long_sequence_init_args": {},
+  "long_sequence_strategy_name": null,
+  "long_sequence_strategy_type": null,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pad_token_id": 0,
+  "paddlenlp_version": "3.0.0b4.post20250825",
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_scaling_factor": 1.0,
+  "rope_scaling_type": null,
+  "rope_theta": 10000.0,
+  "seq_length": 2048,
+  "tie_word_embeddings": false,
+  "use_flash_attention_for_generation": false,
+  "use_last_token_for_generation": false,
+  "use_long_sequence_strategies": false,
+  "vocab_size": 32000
+}

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/lora_config.json

@@ -0,0 +1,32 @@
+{
+  "base_model_name_or_path": null,
+  "do_qat": false,
+  "dtype": "float16",
+  "enable_lora_list": null,
+  "head_dim": null,
+  "lora_alpha": 64,
+  "lora_dropout": 0.1,
+  "lora_plus_scale": 1.0,
+  "lora_use_mixer": false,
+  "loraga": false,
+  "lorapro": false,
+  "merge_weights": false,
+  "pissa": false,
+  "r": 32,
+  "rslora": false,
+  "scaling": 2.0,
+  "target_modules": [
+    ".*q_proj$",
+    ".*k_proj$",
+    ".*v_proj$",
+    ".*o_proj$",
+    ".*down_proj$",
+    ".*up_proj$",
+    ".*gate_proj$"
+  ],
+  "tensor_parallel_degree": 1,
+  "trainable_bias": null,
+  "trainable_modules": null,
+  "use_mora": false,
+  "use_quick_lora": false
+}

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/lora_model_state.pdparams

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c335dd986f620acf65a16100a7e3afda69929957c49e61cd04991171ee838830
+size 479881234

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/sentencepiece.bpe.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/special_tokens_map.json

@@ -0,0 +1 @@
+{"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false}, "pad_token": "<unk>"}

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/tokenizer_config.json

@@ -0,0 +1 @@
+{"unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false, "__type": "AddedToken"}, "bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "special": false, "__type": "AddedToken"}, "add_bos_token": true, "add_eos_token": false, "sp_model_kwargs": {}, "decode_with_prefix_space": false, "clean_up_tokenization_spaces": false, "legacy": true, "model_max_length": 1000000000000000019884624838656, "pad_token": null, "name_or_path": "/mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16", "from_slow": true, "_from_auto": true, "added_tokens_decoder": {}, "tokenizer_class": "LlamaTokenizer"}

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/train_repllama-v1-7b-lora-passage-merge-then-to-paddle-float16_mmarco.jsonl.log

@@ -0,0 +1,739 @@
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md
+  warnings.warn(warning_message)
+LAUNCH INFO 2025-11-01 03:04:46,011 -----------  Configuration  ----------------------
+LAUNCH INFO 2025-11-01 03:04:46,011 auto_cluster_config: 0
+LAUNCH INFO 2025-11-01 03:04:46,011 auto_parallel_config: None
+LAUNCH INFO 2025-11-01 03:04:46,011 auto_tuner_json: None
+LAUNCH INFO 2025-11-01 03:04:46,011 devices: 0,1,2,3
+LAUNCH INFO 2025-11-01 03:04:46,011 elastic_level: -1
+LAUNCH INFO 2025-11-01 03:04:46,011 elastic_timeout: 30
+LAUNCH INFO 2025-11-01 03:04:46,011 enable_gpu_log: True
+LAUNCH INFO 2025-11-01 03:04:46,011 gloo_port: 6767
+LAUNCH INFO 2025-11-01 03:04:46,011 host: None
+LAUNCH INFO 2025-11-01 03:04:46,011 ips: None
+LAUNCH INFO 2025-11-01 03:04:46,011 job_id: default
+LAUNCH INFO 2025-11-01 03:04:46,011 legacy: False
+LAUNCH INFO 2025-11-01 03:04:46,011 log_dir: log
+LAUNCH INFO 2025-11-01 03:04:46,011 log_level: INFO
+LAUNCH INFO 2025-11-01 03:04:46,011 log_overwrite: False
+LAUNCH INFO 2025-11-01 03:04:46,011 master: None
+LAUNCH INFO 2025-11-01 03:04:46,011 max_restart: 3
+LAUNCH INFO 2025-11-01 03:04:46,011 nnodes: 1
+LAUNCH INFO 2025-11-01 03:04:46,011 nproc_per_node: None
+LAUNCH INFO 2025-11-01 03:04:46,011 rank: -1
+LAUNCH INFO 2025-11-01 03:04:46,011 run_mode: collective
+LAUNCH INFO 2025-11-01 03:04:46,011 server_num: None
+LAUNCH INFO 2025-11-01 03:04:46,032 servers: 
+LAUNCH INFO 2025-11-01 03:04:46,032 sort_ip: False
+LAUNCH INFO 2025-11-01 03:04:46,032 start_port: 6070
+LAUNCH INFO 2025-11-01 03:04:46,032 trainer_num: None
+LAUNCH INFO 2025-11-01 03:04:46,032 trainers: 
+LAUNCH INFO 2025-11-01 03:04:46,032 training_script: train.py
+LAUNCH INFO 2025-11-01 03:04:46,032 training_script_args: ['--do_train', '--query_instruction_for_retrieval', 'query: ', '--passage_instruction_for_retrieval', '', '--model_name_or_path', '/mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16', '--output_dir', 'tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16', '--save_steps', '999999999', '--train_data', './data/mmarco.jsonl', '--fp16_opt_level', 'O2', '--fp16', '--per_device_train_batch_size', '1', '--gradient_accumulation_steps', '32', '--recompute', '--train_group_size', '4', '--learning_rate', '1e-4', '--query_max_len', '128', '--passage_max_len', '4096', '--num_train_epochs', '1', '--logging_steps', '1', '--overwrite_output_dir', '--negatives_cross_device', '--warmup_steps', '10', '--max_steps', '100', '--do_train', '--fine_tune_type', 'lora', '--sentence_pooling_method', 'last', '--sharding', 'stage3 offload', '--use_flash_attention', '--temperature', '0.01']
+LAUNCH INFO 2025-11-01 03:04:46,032 with_gloo: 1
+LAUNCH INFO 2025-11-01 03:04:46,032 --------------------------------------------------
+LAUNCH INFO 2025-11-01 03:04:46,051 Job: default, mode collective, replicas 1[1:1], elastic False
+LAUNCH INFO 2025-11-01 03:04:46,117 Run Pod: ueytyy, replicas 4, status ready
+LAUNCH INFO 2025-11-01 03:04:46,617 Watching Pod: ueytyy, replicas 4, status running
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/utils/cpp_extension/extension_utils.py:718: UserWarning: No ccache found. Please be aware that recompiling all source files may be required. You can download and install ccache from: https://github.com/ccache/ccache/blob/master/doc/INSTALL.md
+  warnings.warn(warning_message)
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/_distutils_hack/__init__.py:30: UserWarning: Setuptools is replacing distutils. Support for replacing an already imported distutils is deprecated. In the future, this condition will fail. Register concerns at https://github.com/pypa/setuptools/issues/new?template=distutils-deprecation.yml
+  warnings.warn(
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddlenlp/trainer/training_args.py:1205: UserWarning: `offload` is not supported NOW!
+  warnings.warn("`offload` is not supported NOW!")
+[2025-11-01 03:04:51,354] [    INFO] distributed_strategy.py:335 - distributed strategy initialized
+======================= Modified FLAGS detected =======================
+FLAGS(name='FLAGS_cublas_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cublas/lib', default_value='')
+FLAGS(name='FLAGS_cuda_cccl_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cuda_cccl/include/', default_value='')
+FLAGS(name='FLAGS_cusolver_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cusolver/lib', default_value='')
+FLAGS(name='FLAGS_cupti_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cuda_cupti/lib', default_value='')
+FLAGS(name='FLAGS_cudnn_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cudnn/lib', default_value='')
+FLAGS(name='FLAGS_nvidia_package_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia', default_value='')
+FLAGS(name='FLAGS_selected_gpus', current_value='0', default_value='')
+FLAGS(name='FLAGS_nccl_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/nccl/lib', default_value='')
+FLAGS(name='FLAGS_enable_pir_in_executor', current_value=True, default_value=False)
+FLAGS(name='FLAGS_curand_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/curand/lib', default_value='')
+FLAGS(name='FLAGS_cusparse_dir', current_value='/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/../nvidia/cusparse/lib', default_value='')
+=======================================================================
+I1101 03:04:51.356127 43891 tcp_store.cc:336] create libuv server at port: 57671
+I1101 03:04:51.356886 43891 tcp_utils.cc:132] Successfully connected to 172.17.0.5:57671
+I1101 03:04:54.802934 43891 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
+I1101 03:04:54.802994 43891 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
+[2025-11-01 03:04:54,803] [    INFO] topology.py:526 - Total 4 pipe comm group(s) create successfully!
+W1101 03:04:54.868762 43891 gpu_resources.cc:114] Please NOTE: device: 0, GPU Compute Capability: 8.0, Driver API Version: 12.8, Runtime API Version: 12.8
+/mnt/141nfs/zhongtianyun2023/miniconda3/envs/paddle_0905/lib/python3.10/site-packages/paddle/distributed/communication/group.py:145: UserWarning: Current global rank 0 is not in group _default_pg10
+  warnings.warn(
+[2025-11-01 03:04:54,871] [    INFO] topology.py:526 - Total 4 data comm group(s) create successfully!
+[2025-11-01 03:04:54,871] [    INFO] topology.py:526 - Total 4 model comm group(s) create successfully!
+I1101 03:04:54.871744 43891 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
+I1101 03:04:54.871753 43891 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
+[2025-11-01 03:04:54,871] [    INFO] topology.py:526 - Total 1 sharding comm group(s) create successfully!
+I1101 03:04:54.871822 43891 process_group_nccl.cc:154] ProcessGroupNCCL pg_timeout_ 1800000
+I1101 03:04:54.871829 43891 process_group_nccl.cc:155] ProcessGroupNCCL nccl_comm_init_option_ 0
+[2025-11-01 03:04:54,871] [    INFO] topology.py:440 - HybridParallelInfo: rank_id: 0, mp_degree: 1, sharding_degree: 4, pp_degree: 1, dp_degree: 1, sep_degree: 1, mp_group: [0],  sharding_group: [0, 1, 2, 3], pp_group: [0], dp_group: [0], sep:group: None, check/clip group: [0, 1, 2, 3]
+[2025-11-01 03:04:54,871] [    INFO] -     +==============================================================================+
+    |                                                                              |
+    |                         DistributedStrategy Overview                         |
+    |                                                                              |
+    +==============================================================================+
+    |                        a_sync=True <-> a_sync_configs                        |
+    +------------------------------------------------------------------------------+
+    |                               k_steps                    -1                  |
+    |                     max_merge_var_num                    1                   |
+    |                       send_queue_size                    16                  |
+    |               independent_recv_thread                  False                 |
+    |         min_send_grad_num_before_recv                    1                   |
+    |                      thread_pool_size                    1                   |
+    |                       send_wait_times                    1                   |
+    |               runtime_split_send_recv                  False                 |
+    |                        launch_barrier                   True                 |
+    |             heter_worker_device_guard                   cpu                  |
+    |                        lr_decay_steps                    10                  |
+    |                            use_ps_gpu                    0                   |
+    |                         use_gpu_graph                    0                   |
+    +==============================================================================+
+    |                    Environment Flags, Communication Flags                    |
+    +------------------------------------------------------------------------------+
+    |                                  mode                    1                   |
+    |                               elastic                  False                 |
+    |                                  auto                  False                 |
+    |                   sync_nccl_allreduce                   True                 |
+    |                         nccl_comm_num                    1                   |
+    |            use_hierarchical_allreduce                  False                 |
+    |   hierarchical_allreduce_inter_nranks                    1                   |
+    |                       sync_batch_norm                  False                 |
+    |                   fuse_all_reduce_ops                   True                 |
+    |                  fuse_grad_size_in_MB                    32                  |
+    |              fuse_grad_size_in_TFLOPS                   50.0                 |
+    |               cudnn_exhaustive_search                  False                 |
+    |             conv_workspace_size_limit                   512                  |
+    |    cudnn_batchnorm_spatial_persistent                  False                 |
+    |                        fp16_allreduce                  False                 |
+    |               last_comm_group_size_MB                   1.0                  |
+    |                find_unused_parameters                  False                 |
+    |            without_graph_optimization                   True                 |
+    |                 fuse_grad_size_in_num                    8                   |
+    |                 calc_comm_same_stream                  False                 |
+    |                                   asp                  False                 |
+    |                       fuse_grad_merge                  False                 |
+    |                             semi_auto                  False                 |
+    |                            adam_d2sum                  False                 |
+    |                           auto_search                  False                 |
+    |                        heter_ccl_mode                  False                 |
+    |                         is_fl_ps_mode                  False                 |
+    |                      with_coordinator                  False                 |
+    |                            split_data                   True                 |
+    |                  downpour_table_param                    []                  |
+    |                       fs_client_param                                        |
+    +==============================================================================+
+    |                                Build Strategy                                |
+    +------------------------------------------------------------------------------+
+    |              fuse_elewise_add_act_ops                  False                 |
+    |                       fuse_bn_act_ops                  False                 |
+    |              fuse_relu_depthwise_conv                  False                 |
+    |                    fuse_broadcast_ops                  False                 |
+    |                fuse_all_optimizer_ops                  False                 |
+    |                        enable_inplace                  False                 |
+    |     enable_backward_optimizer_op_deps                   True                 |
+    |                 cache_runtime_context                  False                 |
+    |                   fuse_bn_add_act_ops                   True                 |
+    |                    enable_auto_fusion                  False                 |
+    |                          enable_addto                  False                 |
+    |              allow_cuda_graph_capture                  False                 |
+    |                       reduce_strategy                    0                   |
+    |                    fuse_gemm_epilogue                  False                 |
+    |                   debug_graphviz_path                                        |
+    |                       fused_attention                  False                 |
+    |                     fused_feedforward                  False                 |
+    |            fuse_dot_product_attention                  False                 |
+    |                          fuse_resunit                  False                 |
+    +==============================================================================+
+
+[2025-11-01 03:04:54,872] [    INFO] - The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).
+[2025-11-01 03:04:54,872] [ WARNING] - Process rank: 0, device: gpu, distributed training: True, 16-bits training: True
+[2025-11-01 03:04:54,873] [    INFO] - Training/evaluation parameters RetrieverTrainingArguments(
+adam_beta1=0.9,
+adam_beta2=0.999,
+adam_epsilon=1e-08,
+amp_custom_black_list=None,
+amp_custom_white_list=None,
+amp_master_grad=False,
+auto_parallel_resume_form_hybrid_parallel=False,
+bf16=False,
+bf16_full_eval=False,
+ckpt_quant_stage=O0,
+context_parallel_degree=1,
+count_trained_tokens=False,
+data_parallel_config=,
+dataloader_drop_last=False,
+dataloader_num_workers=0,
+dataloader_shuffle=True,
+ddp_find_unused_parameters=None,
+device=gpu,
+disable_tqdm=False,
+distributed_dataloader=False,
+do_eval=False,
+do_export=False,
+do_predict=False,
+do_train=True,
+enable_auto_parallel=False,
+enable_zero_cost_checkpoint=False,
+eval_accumulation_steps=None,
+eval_steps=None,
+evaluation_strategy=IntervalStrategy.NO,
+expert_max_capacity=4294967296,
+expert_min_capacity=1,
+expert_parallel_degree=1,
+expert_tensor_parallel_degree=1,
+fine_tune_type=lora,
+fix_position_embedding=False,
+flash_device_save_steps=0,
+flatten_param_grads=False,
+force_reshard_pp=False,
+fp16=True,
+fp16_full_eval=False,
+fp16_opt_level=O2,
+fuse_sequence_parallel_allreduce=False,
+gradient_accumulation_steps=32,
+greater_is_better=None,
+hybrid_parallel_topo_order=pp_first,
+ignore_data_skip=False,
+ignore_load_lr_and_optim=False,
+ignore_save_lr_and_optim=False,
+label_names=None,
+lazy_data_processing=True,
+learning_rate=0.0001,
+load_best_model_at_end=False,
+load_sharded_model=False,
+local_rank=0,
+log_on_each_node=True,
+logging_dir=tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/runs/Nov01_03-04-51_b63e119a1648,
+logging_first_step=False,
+logging_steps=1,
+logging_strategy=IntervalStrategy.STEPS,
+lr_end=1e-07,
+lr_scheduler_type=SchedulerType.LINEAR,
+margin=0.2,
+matryoshka_dims=[64, 128, 256, 512, 768],
+matryoshka_loss_weights=[1, 1, 1, 1, 1],
+max_evaluate_steps=-1,
+max_grad_norm=1.0,
+max_steps=100,
+metric_for_best_model=None,
+metrics_output_path=None,
+min_lr=0.0,
+minimum_eval_times=None,
+nccl_comm_group_config=None,
+negatives_cross_device=True,
+no_cuda=False,
+num_cycles=0.5,
+num_train_epochs=1.0,
+offload_optim=False,
+optim=OptimizerNames.ADAMW,
+ordered_save_group_size=0,
+output_dir=tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16,
+output_signal_dir=tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16,
+overwrite_output_dir=True,
+pad_token_id=0,
+past_index=-1,
+pdc_download_ckpt=False,
+pdc_download_timeout=300,
+per_device_eval_batch_size=8,
+per_device_train_batch_size=1,
+pipeline_parallel_config=,
+pipeline_parallel_degree=1,
+power=1.0,
+prediction_loss_only=False,
+recompute=True,
+refined_recompute={},
+release_grads=False,
+remove_unused_columns=True,
+report_to=['visualdl'],
+resume_from_checkpoint=None,
+run_name=tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16,
+save_on_each_node=False,
+save_rng_states=True,
+save_sharded_model=False,
+save_sharding_stage1_model_include_freeze_params=False,
+save_steps=999999999,
+save_strategy=IntervalStrategy.STEPS,
+save_tokenizer=True,
+save_total_limit=None,
+scale_loss=32768,
+seed=42,
+sentence_pooling_method=last,
+sep_parallel_degree=1,
+sequence_parallel=False,
+sequence_parallel_config=,
+sharding=[<ShardingOption.FULL_SHARD: 'stage3'>, <ShardingOption.OFFLOAD: 'offload'>],
+sharding_comm_buffer_size_MB=-1,
+sharding_degree=-1,
+sharding_parallel_config=,
+sharding_parallel_degree=4,
+sharding_parallel_mesh_dimension=dp,
+skip_data_intervals=None,
+skip_memory_metrics=True,
+skip_profile_timer=True,
+split_inputs_sequence_dim=True,
+split_norm_comm=False,
+temperature=0.01,
+tensor_parallel_config=,
+tensor_parallel_degree=1,
+tensorwise_offload_optimizer=False,
+to_static=False,
+unified_checkpoint=False,
+unified_checkpoint_config=,
+use_async_save=False,
+use_expert_parallel=False,
+use_inbatch_neg=False,
+use_lowprecision_moment=False,
+use_matryoshka=False,
+wandb_api_key=None,
+wandb_http_proxy=None,
+warmup_ratio=0.0,
+warmup_steps=10,
+weight_decay=0.0,
+zcc_ema_interval=1,
+zcc_pipeline_hooks_capacity_usage=0.6,
+zcc_save_ema_coef=None,
+zcc_workers_num=3,
+)
+[2025-11-01 03:04:54,873] [    INFO] - Model parameters ModelArguments(model_name_or_path='/mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16', tokenizer_name=None, normalized=True, use_flash_attention=True)
+[2025-11-01 03:04:54,873] [    INFO] - Data parameters DataArguments(train_data='./data/mmarco.jsonl', train_group_size=4, query_max_len=128, passage_max_len=4096, max_example_num_per_dataset=100000000, query_instruction_for_retrieval='query: ', passage_instruction_for_retrieval='')
+[2025-11-01 03:04:54,874] [    INFO] - The global seed is set to 42, local seed is set to 46 and random seed is set to 42.
+[2025-11-01 03:04:54,886] [    INFO] - We are using <class 'paddlenlp.transformers.llama.modeling.LlamaModel'> to load '/mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16'.
+[2025-11-01 03:04:54,886] [    INFO] - Loading configuration file /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/config.json
+[2025-11-01 03:04:54,888] [    INFO] - Loading weights file /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/model_state.pdparams.index.json
+file: /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/model_state.pdparams.index.json is paddle weight.
+
+Loading checkpoint shards:   0%|          | 0/3 [00:00<?, ?it/s]
+Loading checkpoint shards:  33%|███▎      | 1/3 [00:08<00:17,  8.92s/it]
+Loading checkpoint shards:  67%|██████▋   | 2/3 [00:17<00:08,  8.89s/it]
+Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  7.71s/it]
+Loading checkpoint shards: 100%|██████████| 3/3 [00:24<00:00,  8.03s/it]
+[2025-11-01 03:05:19,407] [ WARNING] - Some weights of the model checkpoint at /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16 were not used when initializing LlamaModel: ['lm_head.weight']
+- This IS expected if you are initializing LlamaModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
+- This IS NOT expected if you are initializing LlamaModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
+[2025-11-01 03:05:19,407] [    INFO] - All the weights of LlamaModel were initialized from the model checkpoint at /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaModel for predictions without further training.
+[2025-11-01 03:05:19,410] [    INFO] - Loading configuration file /mnt/141nfs/lizhuoqun/hf_models/llama-2-7b/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/config.json
+[2025-11-01 03:05:19,793] [ WARNING] - Reset tensor_parallel_degree of lora_config to 1.
+[2025-11-01 03:05:19,793] [    INFO] - Mark only lora and trainable_module as trainable.
+[2025-11-01 03:05:19,821] [   DEBUG] - Frozen parameters: 1.98e+10 || Trainable parameters:2.40e+08 || Total parameters:2.01e+10|| Trainable:1.20%
+[2025-11-01 03:05:19,859] [    INFO] - The global seed is set to 42, local seed is set to 46 and random seed is set to 42.
+[2025-11-01 03:05:19,917] [    INFO] - max_steps is given, it will override any value given in num_train_epochs
+[2025-11-01 03:05:19,917] [    INFO] - Using half precision
+[2025-11-01 03:05:19,941] [   DEBUG] - ============================================================
+[2025-11-01 03:05:19,941] [   DEBUG] -     Training Configuration Arguments    
+[2025-11-01 03:05:19,941] [   DEBUG] - paddle commit id              : dc312219f1498234f178a3b6a93e9f1f4ddebcb8
+[2025-11-01 03:05:19,941] [   DEBUG] - paddlenlp commit id           : 0bb61071d051adf6207199d6493b08079d58f8b2.dirty
+[2025-11-01 03:05:19,942] [   DEBUG] - _no_sync_in_gradient_accumulation: True
+[2025-11-01 03:05:19,942] [   DEBUG] - adam_beta1                    : 0.9
+[2025-11-01 03:05:19,942] [   DEBUG] - adam_beta2                    : 0.999
+[2025-11-01 03:05:19,942] [   DEBUG] - adam_epsilon                  : 1e-08
+[2025-11-01 03:05:19,942] [   DEBUG] - amp_custom_black_list         : None
+[2025-11-01 03:05:19,942] [   DEBUG] - amp_custom_white_list         : None
+[2025-11-01 03:05:19,942] [   DEBUG] - amp_master_grad               : False
+[2025-11-01 03:05:19,942] [   DEBUG] - auto_parallel_resume_form_hybrid_parallel: False
+[2025-11-01 03:05:19,942] [   DEBUG] - bf16                          : False
+[2025-11-01 03:05:19,942] [   DEBUG] - bf16_full_eval                : False
+[2025-11-01 03:05:19,942] [   DEBUG] - ckpt_quant_stage              : O0
+[2025-11-01 03:05:19,942] [   DEBUG] - context_parallel_degree       : 1
+[2025-11-01 03:05:19,942] [   DEBUG] - count_trained_tokens          : False
+[2025-11-01 03:05:19,942] [   DEBUG] - current_device                : gpu:0
+[2025-11-01 03:05:19,942] [   DEBUG] - data_parallel_config          : 
+[2025-11-01 03:05:19,942] [   DEBUG] - data_parallel_degree          : 1
+[2025-11-01 03:05:19,942] [   DEBUG] - data_parallel_rank            : 0
+[2025-11-01 03:05:19,942] [   DEBUG] - dataloader_drop_last          : False
+[2025-11-01 03:05:19,942] [   DEBUG] - dataloader_num_workers        : 0
+[2025-11-01 03:05:19,943] [   DEBUG] - dataloader_shuffle            : True
+[2025-11-01 03:05:19,943] [   DEBUG] - dataset_rank                  : 0
+[2025-11-01 03:05:19,943] [   DEBUG] - dataset_world_size            : 4
+[2025-11-01 03:05:19,943] [   DEBUG] - ddp_find_unused_parameters    : None
+[2025-11-01 03:05:19,943] [   DEBUG] - device                        : gpu
+[2025-11-01 03:05:19,943] [   DEBUG] - disable_tqdm                  : False
+[2025-11-01 03:05:19,943] [   DEBUG] - distributed_dataloader        : False
+[2025-11-01 03:05:19,943] [   DEBUG] - do_eval                       : False
+[2025-11-01 03:05:19,943] [   DEBUG] - do_export                     : False
+[2025-11-01 03:05:19,943] [   DEBUG] - do_predict                    : False
+[2025-11-01 03:05:19,943] [   DEBUG] - do_train                      : True
+[2025-11-01 03:05:19,943] [   DEBUG] - enable_auto_parallel          : False
+[2025-11-01 03:05:19,943] [   DEBUG] - enable_zero_cost_checkpoint   : False
+[2025-11-01 03:05:19,943] [   DEBUG] - eval_accumulation_steps       : None
+[2025-11-01 03:05:19,943] [   DEBUG] - eval_batch_size               : 8
+[2025-11-01 03:05:19,943] [   DEBUG] - eval_steps                    : None
+[2025-11-01 03:05:19,943] [   DEBUG] - evaluation_strategy           : IntervalStrategy.NO
+[2025-11-01 03:05:19,943] [   DEBUG] - expert_max_capacity           : 4294967296
+[2025-11-01 03:05:19,943] [   DEBUG] - expert_min_capacity           : 1
+[2025-11-01 03:05:19,944] [   DEBUG] - expert_parallel_degree        : 1
+[2025-11-01 03:05:19,944] [   DEBUG] - expert_tensor_parallel_degree : 1
+[2025-11-01 03:05:19,944] [   DEBUG] - fine_tune_type                : lora
+[2025-11-01 03:05:19,944] [   DEBUG] - fix_position_embedding        : False
+[2025-11-01 03:05:19,944] [   DEBUG] - flash_device_save_steps       : 0
+[2025-11-01 03:05:19,944] [   DEBUG] - flatten_param_grads           : False
+[2025-11-01 03:05:19,944] [   DEBUG] - force_reshard_pp              : False
+[2025-11-01 03:05:19,944] [   DEBUG] - fp16                          : True
+[2025-11-01 03:05:19,944] [   DEBUG] - fp16_full_eval                : False
+[2025-11-01 03:05:19,944] [   DEBUG] - fp16_opt_level                : O2
+[2025-11-01 03:05:19,944] [   DEBUG] - fuse_sequence_parallel_allreduce: False
+[2025-11-01 03:05:19,944] [   DEBUG] - gradient_accumulation_steps   : 32
+[2025-11-01 03:05:19,944] [   DEBUG] - greater_is_better             : None
+[2025-11-01 03:05:19,944] [   DEBUG] - hybrid_parallel_topo_order    : pp_first
+[2025-11-01 03:05:19,944] [   DEBUG] - ignore_data_skip              : False
+[2025-11-01 03:05:19,944] [   DEBUG] - ignore_load_lr_and_optim      : False
+[2025-11-01 03:05:19,944] [   DEBUG] - ignore_save_lr_and_optim      : False
+[2025-11-01 03:05:19,944] [   DEBUG] - label_names                   : None
+[2025-11-01 03:05:19,944] [   DEBUG] - lazy_data_processing          : True
+[2025-11-01 03:05:19,944] [   DEBUG] - learning_rate                 : 0.0001
+[2025-11-01 03:05:19,945] [   DEBUG] - load_best_model_at_end        : False
+[2025-11-01 03:05:19,945] [   DEBUG] - load_sharded_model            : False
+[2025-11-01 03:05:19,945] [   DEBUG] - local_process_index           : 0
+[2025-11-01 03:05:19,945] [   DEBUG] - local_rank                    : 0
+[2025-11-01 03:05:19,945] [   DEBUG] - log_level                     : -1
+[2025-11-01 03:05:19,945] [   DEBUG] - log_level_replica             : -1
+[2025-11-01 03:05:19,945] [   DEBUG] - log_on_each_node              : True
+[2025-11-01 03:05:19,945] [   DEBUG] - logging_dir                   : tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/runs/Nov01_03-04-51_b63e119a1648
+[2025-11-01 03:05:19,945] [   DEBUG] - logging_first_step            : False
+[2025-11-01 03:05:19,945] [   DEBUG] - logging_steps                 : 1
+[2025-11-01 03:05:19,945] [   DEBUG] - logging_strategy              : IntervalStrategy.STEPS
+[2025-11-01 03:05:19,945] [   DEBUG] - logical_process_index         : 0
+[2025-11-01 03:05:19,945] [   DEBUG] - lr_end                        : 1e-07
+[2025-11-01 03:05:19,945] [   DEBUG] - lr_scheduler_type             : SchedulerType.LINEAR
+[2025-11-01 03:05:19,945] [   DEBUG] - margin                        : 0.2
+[2025-11-01 03:05:19,945] [   DEBUG] - matryoshka_dims               : [64, 128, 256, 512, 768]
+[2025-11-01 03:05:19,945] [   DEBUG] - matryoshka_loss_weights       : [1, 1, 1, 1, 1]
+[2025-11-01 03:05:19,945] [   DEBUG] - max_evaluate_steps            : -1
+[2025-11-01 03:05:19,945] [   DEBUG] - max_grad_norm                 : 1.0
+[2025-11-01 03:05:19,946] [   DEBUG] - max_steps                     : 100
+[2025-11-01 03:05:19,946] [   DEBUG] - metric_for_best_model         : None
+[2025-11-01 03:05:19,946] [   DEBUG] - metrics_output_path           : None
+[2025-11-01 03:05:19,946] [   DEBUG] - min_lr                        : 0.0
+[2025-11-01 03:05:19,946] [   DEBUG] - minimum_eval_times            : None
+[2025-11-01 03:05:19,946] [   DEBUG] - nccl_comm_group_config        : None
+[2025-11-01 03:05:19,946] [   DEBUG] - negatives_cross_device        : True
+[2025-11-01 03:05:19,946] [   DEBUG] - no_cuda                       : False
+[2025-11-01 03:05:19,946] [   DEBUG] - num_cycles                    : 0.5
+[2025-11-01 03:05:19,946] [   DEBUG] - num_train_epochs              : 1.0
+[2025-11-01 03:05:19,946] [   DEBUG] - offload_optim                 : False
+[2025-11-01 03:05:19,946] [   DEBUG] - optim                         : OptimizerNames.ADAMW
+[2025-11-01 03:05:19,946] [   DEBUG] - optimizer_name_suffix         : shard00
+[2025-11-01 03:05:19,946] [   DEBUG] - ordered_save_group_size       : 0
+[2025-11-01 03:05:19,946] [   DEBUG] - output_dir                    : tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16
+[2025-11-01 03:05:19,946] [   DEBUG] - output_signal_dir             : tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16
+[2025-11-01 03:05:19,946] [   DEBUG] - overwrite_output_dir          : True
+[2025-11-01 03:05:19,946] [   DEBUG] - pad_token_id                  : 0
+[2025-11-01 03:05:19,946] [   DEBUG] - past_index                    : -1
+[2025-11-01 03:05:19,947] [   DEBUG] - pdc_download_ckpt             : False
+[2025-11-01 03:05:19,947] [   DEBUG] - pdc_download_timeout          : 300
+[2025-11-01 03:05:19,947] [   DEBUG] - per_device_eval_batch_size    : 8
+[2025-11-01 03:05:19,947] [   DEBUG] - per_device_train_batch_size   : 1
+[2025-11-01 03:05:19,947] [   DEBUG] - pipeline_parallel_config      : 
+[2025-11-01 03:05:19,947] [   DEBUG] - pipeline_parallel_degree      : 1
+[2025-11-01 03:05:19,947] [   DEBUG] - pipeline_parallel_rank        : 0
+[2025-11-01 03:05:19,947] [   DEBUG] - power                         : 1.0
+[2025-11-01 03:05:19,947] [   DEBUG] - prediction_loss_only          : False
+[2025-11-01 03:05:19,947] [   DEBUG] - process_index                 : 0
+[2025-11-01 03:05:19,947] [   DEBUG] - recompute                     : True
+[2025-11-01 03:05:19,947] [   DEBUG] - refined_recompute             : {}
+[2025-11-01 03:05:19,947] [   DEBUG] - release_grads                 : False
+[2025-11-01 03:05:19,947] [   DEBUG] - remove_unused_columns         : True
+[2025-11-01 03:05:19,947] [   DEBUG] - report_to                     : ['visualdl']
+[2025-11-01 03:05:19,947] [   DEBUG] - resume_from_checkpoint        : None
+[2025-11-01 03:05:19,947] [   DEBUG] - run_name                      : tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16
+[2025-11-01 03:05:19,947] [   DEBUG] - save_on_each_node             : False
+[2025-11-01 03:05:19,947] [   DEBUG] - save_rng_states               : True
+[2025-11-01 03:05:19,947] [   DEBUG] - save_sharded_model            : False
+[2025-11-01 03:05:19,948] [   DEBUG] - save_sharding_stage1_model_include_freeze_params: False
+[2025-11-01 03:05:19,948] [   DEBUG] - save_steps                    : 999999999
+[2025-11-01 03:05:19,948] [   DEBUG] - save_strategy                 : IntervalStrategy.STEPS
+[2025-11-01 03:05:19,948] [   DEBUG] - save_tokenizer                : True
+[2025-11-01 03:05:19,948] [   DEBUG] - save_total_limit              : None
+[2025-11-01 03:05:19,948] [   DEBUG] - scale_loss                    : 32768
+[2025-11-01 03:05:19,948] [   DEBUG] - seed                          : 42
+[2025-11-01 03:05:19,948] [   DEBUG] - sentence_pooling_method       : last
+[2025-11-01 03:05:19,948] [   DEBUG] - sep_parallel_degree           : 1
+[2025-11-01 03:05:19,948] [   DEBUG] - sequence_parallel             : False
+[2025-11-01 03:05:19,948] [   DEBUG] - sequence_parallel_config      : 
+[2025-11-01 03:05:19,948] [   DEBUG] - sharding                      : [<ShardingOption.FULL_SHARD: 'stage3'>, <ShardingOption.OFFLOAD: 'offload'>]
+[2025-11-01 03:05:19,948] [   DEBUG] - sharding_comm_buffer_size_MB  : -1
+[2025-11-01 03:05:19,948] [   DEBUG] - sharding_degree               : -1
+[2025-11-01 03:05:19,948] [   DEBUG] - sharding_parallel_config      : 
+[2025-11-01 03:05:19,948] [   DEBUG] - sharding_parallel_degree      : 4
+[2025-11-01 03:05:19,948] [   DEBUG] - sharding_parallel_mesh_dimension: dp
+[2025-11-01 03:05:19,948] [   DEBUG] - sharding_parallel_rank        : 0
+[2025-11-01 03:05:19,948] [   DEBUG] - should_load_dataset           : True
+[2025-11-01 03:05:19,949] [   DEBUG] - should_load_sharding_stage1_model: False
+[2025-11-01 03:05:19,949] [   DEBUG] - should_log                    : True
+[2025-11-01 03:05:19,949] [   DEBUG] - should_save                   : True
+[2025-11-01 03:05:19,949] [   DEBUG] - should_save_model_state       : True
+[2025-11-01 03:05:19,949] [   DEBUG] - should_save_model_with_tensor_fusion: False
+[2025-11-01 03:05:19,949] [   DEBUG] - should_save_sharding_stage1_model: False
+[2025-11-01 03:05:19,949] [   DEBUG] - skip_data_intervals           : None
+[2025-11-01 03:05:19,949] [   DEBUG] - skip_memory_metrics           : True
+[2025-11-01 03:05:19,949] [   DEBUG] - skip_profile_timer            : True
+[2025-11-01 03:05:19,949] [   DEBUG] - split_inputs_sequence_dim     : True
+[2025-11-01 03:05:19,949] [   DEBUG] - split_norm_comm               : False
+[2025-11-01 03:05:19,949] [   DEBUG] - temperature                   : 0.01
+[2025-11-01 03:05:19,949] [   DEBUG] - tensor_parallel_config        : 
+[2025-11-01 03:05:19,949] [   DEBUG] - tensor_parallel_degree        : 1
+[2025-11-01 03:05:19,949] [   DEBUG] - tensor_parallel_rank          : 0
+[2025-11-01 03:05:19,949] [   DEBUG] - tensorwise_offload_optimizer  : False
+[2025-11-01 03:05:19,949] [   DEBUG] - to_static                     : False
+[2025-11-01 03:05:19,949] [   DEBUG] - train_batch_size              : 1
+[2025-11-01 03:05:19,949] [   DEBUG] - unified_checkpoint            : False
+[2025-11-01 03:05:19,949] [   DEBUG] - unified_checkpoint_config     : 
+[2025-11-01 03:05:19,950] [   DEBUG] - use_async_save                : False
+[2025-11-01 03:05:19,950] [   DEBUG] - use_expert_parallel           : False
+[2025-11-01 03:05:19,950] [   DEBUG] - use_hybrid_parallel           : True
+[2025-11-01 03:05:19,950] [   DEBUG] - use_inbatch_neg               : False
+[2025-11-01 03:05:19,950] [   DEBUG] - use_lowprecision_moment       : False
+[2025-11-01 03:05:19,950] [   DEBUG] - use_matryoshka                : False
+[2025-11-01 03:05:19,950] [   DEBUG] - wandb_api_key                 : None
+[2025-11-01 03:05:19,950] [   DEBUG] - wandb_http_proxy              : None
+[2025-11-01 03:05:19,950] [   DEBUG] - warmup_ratio                  : 0.0
+[2025-11-01 03:05:19,950] [   DEBUG] - warmup_steps                  : 10
+[2025-11-01 03:05:19,950] [   DEBUG] - weight_decay                  : 0.0
+[2025-11-01 03:05:19,950] [   DEBUG] - weight_name_suffix            : 
+[2025-11-01 03:05:19,950] [   DEBUG] - world_size                    : 4
+[2025-11-01 03:05:19,950] [   DEBUG] - zcc_ema_interval              : 1
+[2025-11-01 03:05:19,950] [   DEBUG] - zcc_pipeline_hooks_capacity_usage: 0.6
+[2025-11-01 03:05:19,950] [   DEBUG] - zcc_save_ema_coef             : None
+[2025-11-01 03:05:19,950] [   DEBUG] - zcc_workers_num               : 3
+[2025-11-01 03:05:19,950] [   DEBUG] - 
+[2025-11-01 03:05:19,950] [    INFO] - Starting training from resume_from_checkpoint : None
+W1101 03:05:20.139665 43891 nccl_comm_context.cc:70] ncclCommInitRankConfigMemOpt is not supported.
+[2025-11-01 03:05:20,737] [ WARNING] group_sharded.py:148 - the input of scaler is None, please ensure the logic of your scaler outside is same as GroupShardedScaler.
+[2025-11-01 03:05:20,743] [ WARNING] group_sharded_stage3.py:195 - While using ClipGradByGlobalNorm in GroupShardedStage3, the grad clip of original optimizer will be changed.
+W1101 03:05:20.747748 43891 nccl_comm_context.cc:70] ncclCommInitRankConfigMemOpt is not supported.
+[2025-11-01 03:05:54,056] [    INFO] - [timelog] checkpoint loading time: 0.00s (2025-11-01 03:05:54) 
+[2025-11-01 03:05:54,058] [    INFO] - ***** Running training *****
+[2025-11-01 03:05:54,058] [    INFO] -   Num examples = 200,001
+[2025-11-01 03:05:54,058] [    INFO] -   Num Epochs = 1
+[2025-11-01 03:05:54,062] [    INFO] -   Instantaneous batch size per device = 1
+[2025-11-01 03:05:54,062] [    INFO] -   Total train batch size (w. parallel, distributed & accumulation) = 128
+[2025-11-01 03:05:54,063] [    INFO] -   Gradient Accumulation steps = 32
+[2025-11-01 03:05:54,063] [    INFO] -   Total optimization steps = 100
+[2025-11-01 03:05:54,066] [    INFO] -   Total num train samples = 12,800
+[2025-11-01 03:05:54,071] [   DEBUG] -   Number of trainable parameters = 79,953,920 (per device)
+
+TrainProcess:   0%|          | 0/100 [00:00<?, ?it/s]W1101 03:05:54.751430 43891 multiply_fwd_func.cc:82] got different data type, run type promotion automatically, this may cause data type been changed.
+W1101 03:06:22.858916 43891 dygraph_functions.cc:94044] got different data type, run type promotion automatically, this may cause data type been changed.
+
+TrainProcess:   1%|          | 1/100 [05:56<9:47:24, 356.01s/it][2025-11-01 03:11:50,167] [    INFO] - loss: 0.27281576, learning_rate: 1e-05, global_step: 1, current_memory_allocated: 12.27591347694397, current_memory_reserved: 70.4673011302948, max_memory_allocated: 57.78868627548218, max_memory_reserved: 70.4673011302948, interval_runtime: 356.0266, interval_samples_per_second: 0.3595, interval_steps_per_second: 0.0028, progress_or_epoch: 0.0006
+
+TrainProcess:   2%|▏         | 2/100 [12:41<10:28:52, 385.02s/it][2025-11-01 03:18:35,525] [    INFO] - loss: 0.26489317, learning_rate: 2e-05, global_step: 2, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.3348, interval_samples_per_second: 0.3158, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0013
+
+TrainProcess:   3%|▎         | 3/100 [19:28<10:38:28, 394.93s/it][2025-11-01 03:25:22,271] [    INFO] - loss: 0.33081675, learning_rate: 3e-05, global_step: 3, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.7202, interval_samples_per_second: 0.3147, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0019
+
+TrainProcess:   4%|▍         | 4/100 [26:14<10:38:59, 399.37s/it][2025-11-01 03:32:08,444] [    INFO] - loss: 0.2576859, learning_rate: 4e-05, global_step: 4, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.1721, interval_samples_per_second: 0.3151, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0026
+
+TrainProcess:   5%|▌         | 5/100 [33:01<10:36:44, 402.15s/it][2025-11-01 03:38:55,513] [    INFO] - loss: 0.22716051, learning_rate: 5e-05, global_step: 5, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.0764, interval_samples_per_second: 0.3144, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0032
+
+TrainProcess:   6%|▌         | 6/100 [39:48<10:32:58, 404.03s/it][2025-11-01 03:45:43,183] [    INFO] - loss: 0.3455947, learning_rate: 6e-05, global_step: 6, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.6721, interval_samples_per_second: 0.314, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0038
+
+TrainProcess:   7%|▋         | 7/100 [46:35<10:27:44, 404.99s/it][2025-11-01 03:52:30,209] [    INFO] - loss: 0.26772273, learning_rate: 7e-05, global_step: 7, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.983, interval_samples_per_second: 0.3145, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0045
+
+TrainProcess:   8%|▊         | 8/100 [53:23<10:22:04, 405.70s/it][2025-11-01 03:59:17,358] [    INFO] - loss: 0.19551414, learning_rate: 8e-05, global_step: 8, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.2231, interval_samples_per_second: 0.3143, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0051
+
+TrainProcess:   9%|▉         | 9/100 [1:00:10<10:16:14, 406.31s/it][2025-11-01 04:06:05,063] [    INFO] - loss: 0.27931464, learning_rate: 9e-05, global_step: 9, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.6456, interval_samples_per_second: 0.314, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0058
+
+TrainProcess:  10%|█         | 10/100 [1:06:58<10:10:12, 406.81s/it][2025-11-01 04:12:53,045] [    INFO] - loss: 0.10694677, learning_rate: 0.0001, global_step: 10, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.9246, interval_samples_per_second: 0.3138, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0064
+
+TrainProcess:  11%|█         | 11/100 [1:13:46<10:03:45, 407.03s/it][2025-11-01 04:19:40,523] [    INFO] - loss: 0.21566278, learning_rate: 9.889e-05, global_step: 11, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.5423, interval_samples_per_second: 0.3141, interval_steps_per_second: 0.0025, progress_or_epoch: 0.007
+
+TrainProcess:  12%|█▏        | 12/100 [1:20:33<9:56:55, 407.00s/it] [2025-11-01 04:26:27,509] [    INFO] - loss: 0.29419184, learning_rate: 9.778e-05, global_step: 12, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.922, interval_samples_per_second: 0.3146, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0077
+
+TrainProcess:  13%|█▎        | 13/100 [1:27:19<9:49:56, 406.86s/it][2025-11-01 04:33:13,985] [    INFO] - loss: 0.18867421, learning_rate: 9.667e-05, global_step: 13, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.5816, interval_samples_per_second: 0.3148, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0083
+
+TrainProcess:  14%|█▍        | 14/100 [1:34:06<9:43:00, 406.76s/it][2025-11-01 04:40:00,453] [    INFO] - loss: 0.12043172, learning_rate: 9.556e-05, global_step: 14, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.4667, interval_samples_per_second: 0.3149, interval_steps_per_second: 0.0025, progress_or_epoch: 0.009
+
+TrainProcess:  15%|█▌        | 15/100 [1:40:51<9:35:44, 406.41s/it][2025-11-01 04:46:46,097] [    INFO] - loss: 0.13413012, learning_rate: 9.444e-05, global_step: 15, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.6027, interval_samples_per_second: 0.3156, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0096
+
+TrainProcess:  16%|█▌        | 16/100 [1:47:37<9:28:32, 406.10s/it][2025-11-01 04:53:31,473] [    INFO] - loss: 0.15476739, learning_rate: 9.333e-05, global_step: 16, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.3761, interval_samples_per_second: 0.3158, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0102
+
+TrainProcess:  17%|█▋        | 17/100 [1:54:24<9:22:07, 406.36s/it][2025-11-01 05:00:18,446] [    INFO] - loss: 0.10692233, learning_rate: 9.222e-05, global_step: 17, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.9967, interval_samples_per_second: 0.3145, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0109
+
+TrainProcess:  18%|█▊        | 18/100 [2:01:11<9:15:41, 406.60s/it][2025-11-01 05:07:05,579] [    INFO] - loss: 0.40895057, learning_rate: 9.111e-05, global_step: 18, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.1305, interval_samples_per_second: 0.3144, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0115
+
+TrainProcess:  19%|█▉        | 19/100 [2:07:59<9:09:43, 407.21s/it][2025-11-01 05:13:54,263] [    INFO] - loss: 0.22335815, learning_rate: 9e-05, global_step: 19, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.6178, interval_samples_per_second: 0.3133, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0122
+
+TrainProcess:  20%|██        | 20/100 [2:14:50<9:04:05, 408.06s/it][2025-11-01 05:20:44,282] [    INFO] - loss: 0.14819849, learning_rate: 8.889e-05, global_step: 20, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.0584, interval_samples_per_second: 0.3122, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0128
+
+TrainProcess:  21%|██        | 21/100 [2:21:40<8:58:18, 408.85s/it][2025-11-01 05:27:34,918] [    INFO] - loss: 0.10684299, learning_rate: 8.778e-05, global_step: 21, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.6716, interval_samples_per_second: 0.3117, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0134
+
+TrainProcess:  22%|██▏       | 22/100 [2:28:29<8:51:37, 408.94s/it][2025-11-01 05:34:24,136] [    INFO] - loss: 0.14283717, learning_rate: 8.667e-05, global_step: 22, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.1728, interval_samples_per_second: 0.3128, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0141
+
+TrainProcess:  23%|██▎       | 23/100 [2:35:19<8:45:14, 409.29s/it][2025-11-01 05:41:14,235] [    INFO] - loss: 0.11052269, learning_rate: 8.556e-05, global_step: 23, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.0805, interval_samples_per_second: 0.3121, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0147
+
+TrainProcess:  24%|██▍       | 24/100 [2:42:10<8:38:42, 409.51s/it][2025-11-01 05:48:04,162] [    INFO] - loss: 0.15822148, learning_rate: 8.444e-05, global_step: 24, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.0229, interval_samples_per_second: 0.3122, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0154
+
+TrainProcess:  25%|██▌       | 25/100 [2:49:00<8:32:08, 409.71s/it][2025-11-01 05:54:54,397] [    INFO] - loss: 0.20399958, learning_rate: 8.333e-05, global_step: 25, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 410.1983, interval_samples_per_second: 0.312, interval_steps_per_second: 0.0024, progress_or_epoch: 0.016
+
+TrainProcess:  26%|██▌       | 26/100 [2:55:50<8:25:20, 409.74s/it][2025-11-01 06:01:44,203] [    INFO] - loss: 0.2004016, learning_rate: 8.222e-05, global_step: 26, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.8053, interval_samples_per_second: 0.3123, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0166
+
+TrainProcess:  27%|██▋       | 27/100 [3:02:39<8:18:19, 409.58s/it][2025-11-01 06:08:33,420] [    INFO] - loss: 0.11004019, learning_rate: 8.111e-05, global_step: 27, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.2179, interval_samples_per_second: 0.3128, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0173
+
+TrainProcess:  28%|██▊       | 28/100 [3:09:27<8:11:09, 409.30s/it][2025-11-01 06:15:22,073] [    INFO] - loss: 0.15227807, learning_rate: 8e-05, global_step: 28, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.6405, interval_samples_per_second: 0.3132, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0179
+
+TrainProcess:  29%|██▉       | 29/100 [3:16:15<8:03:39, 408.72s/it][2025-11-01 06:22:09,442] [    INFO] - loss: 0.16492319, learning_rate: 7.889e-05, global_step: 29, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.3608, interval_samples_per_second: 0.3142, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0186
+
+TrainProcess:  30%|███       | 30/100 [3:23:02<7:56:24, 408.35s/it][2025-11-01 06:28:56,955] [    INFO] - loss: 0.30878437, learning_rate: 7.778e-05, global_step: 30, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.4966, interval_samples_per_second: 0.3141, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0192
+
+TrainProcess:  31%|███       | 31/100 [3:29:49<7:49:08, 407.95s/it][2025-11-01 06:35:43,926] [    INFO] - loss: 0.15092909, learning_rate: 7.667e-05, global_step: 31, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.0082, interval_samples_per_second: 0.3145, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0198
+
+TrainProcess:  32%|███▏      | 32/100 [3:36:36<7:41:51, 407.52s/it][2025-11-01 06:42:30,475] [    INFO] - loss: 0.11128396, learning_rate: 7.556e-05, global_step: 32, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.5133, interval_samples_per_second: 0.3149, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0205
+
+TrainProcess:  33%|███▎      | 33/100 [3:43:23<7:34:52, 407.35s/it][2025-11-01 06:49:17,381] [    INFO] - loss: 0.21856159, learning_rate: 7.444e-05, global_step: 33, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.9415, interval_samples_per_second: 0.3145, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0211
+
+TrainProcess:  34%|███▍      | 34/100 [3:50:09<7:27:42, 407.01s/it][2025-11-01 06:56:03,717] [    INFO] - loss: 0.21674383, learning_rate: 7.333e-05, global_step: 34, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.2376, interval_samples_per_second: 0.3151, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0218
+
+TrainProcess:  35%|███▌      | 35/100 [3:56:56<7:21:05, 407.16s/it][2025-11-01 07:02:51,128] [    INFO] - loss: 0.16016787, learning_rate: 7.222e-05, global_step: 35, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.5021, interval_samples_per_second: 0.3141, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0224
+
+TrainProcess:  36%|███▌      | 36/100 [4:03:44<7:14:33, 407.40s/it][2025-11-01 07:09:39,129] [    INFO] - loss: 0.10064834, learning_rate: 7.111e-05, global_step: 36, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.9519, interval_samples_per_second: 0.3138, interval_steps_per_second: 0.0025, progress_or_epoch: 0.023
+
+TrainProcess:  37%|███▋      | 37/100 [4:10:34<7:08:20, 407.95s/it][2025-11-01 07:16:28,412] [    INFO] - loss: 0.28793156, learning_rate: 7e-05, global_step: 37, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.2433, interval_samples_per_second: 0.3128, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0237
+
+TrainProcess:  38%|███▊      | 38/100 [4:17:22<7:01:33, 407.96s/it][2025-11-01 07:23:16,330] [    INFO] - loss: 0.11859709, learning_rate: 6.889e-05, global_step: 38, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.0018, interval_samples_per_second: 0.3137, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0243
+
+TrainProcess:  39%|███▉      | 39/100 [4:24:09<6:54:36, 407.81s/it][2025-11-01 07:30:03,875] [    INFO] - loss: 0.18987525, learning_rate: 6.778e-05, global_step: 39, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.4877, interval_samples_per_second: 0.3141, interval_steps_per_second: 0.0025, progress_or_epoch: 0.025
+
+TrainProcess:  40%|████      | 40/100 [4:30:57<6:47:42, 407.71s/it][2025-11-01 07:36:51,302] [    INFO] - loss: 0.18106782, learning_rate: 6.667e-05, global_step: 40, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.4683, interval_samples_per_second: 0.3141, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0256
+
+TrainProcess:  41%|████      | 41/100 [4:37:45<6:41:06, 407.90s/it][2025-11-01 07:43:39,626] [    INFO] - loss: 0.28041112, learning_rate: 6.556e-05, global_step: 41, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.3324, interval_samples_per_second: 0.3135, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0262
+
+TrainProcess:  42%|████▏     | 42/100 [4:44:32<6:34:09, 407.75s/it][2025-11-01 07:50:26,967] [    INFO] - loss: 0.17783225, learning_rate: 6.444e-05, global_step: 42, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.3741, interval_samples_per_second: 0.3142, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0269
+
+TrainProcess:  43%|████▎     | 43/100 [4:51:22<6:27:47, 408.20s/it][2025-11-01 07:57:16,317] [    INFO] - loss: 0.12677962, learning_rate: 6.333e-05, global_step: 43, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.2483, interval_samples_per_second: 0.3128, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0275
+
+TrainProcess:  44%|████▍     | 44/100 [4:58:08<6:20:31, 407.70s/it][2025-11-01 08:04:02,788] [    INFO] - loss: 0.25125831, learning_rate: 6.222e-05, global_step: 44, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.5527, interval_samples_per_second: 0.3148, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0282
+
+TrainProcess:  45%|████▌     | 45/100 [5:04:54<6:13:20, 407.28s/it][2025-11-01 08:10:49,131] [    INFO] - loss: 0.18968374, learning_rate: 6.111e-05, global_step: 45, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.3029, interval_samples_per_second: 0.315, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0288
+
+TrainProcess:  46%|████▌     | 46/100 [5:11:43<6:06:49, 407.59s/it][2025-11-01 08:17:37,370] [    INFO] - loss: 0.15617853, learning_rate: 6e-05, global_step: 46, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.2926, interval_samples_per_second: 0.3135, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0294
+
+TrainProcess:  47%|████▋     | 47/100 [5:18:29<5:59:42, 407.21s/it][2025-11-01 08:24:23,741] [    INFO] - loss: 0.05759466, learning_rate: 5.889e-05, global_step: 47, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.3452, interval_samples_per_second: 0.315, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0301
+
+TrainProcess:  48%|████▊     | 48/100 [5:25:15<5:52:42, 406.97s/it][2025-11-01 08:31:10,199] [    INFO] - loss: 0.11871034, learning_rate: 5.778e-05, global_step: 48, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.398, interval_samples_per_second: 0.315, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0307
+
+TrainProcess:  49%|████▉     | 49/100 [5:32:03<5:46:10, 407.26s/it][2025-11-01 08:37:58,042] [    INFO] - loss: 0.06416416, learning_rate: 5.667e-05, global_step: 49, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.9534, interval_samples_per_second: 0.3138, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0314
+
+TrainProcess:  50%|█████     | 50/100 [5:38:51<5:39:26, 407.33s/it][2025-11-01 08:44:45,552] [    INFO] - loss: 0.23245418, learning_rate: 5.556e-05, global_step: 50, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.4873, interval_samples_per_second: 0.3141, interval_steps_per_second: 0.0025, progress_or_epoch: 0.032
+
+TrainProcess:  51%|█████     | 51/100 [5:45:37<5:32:19, 406.93s/it][2025-11-01 08:51:31,610] [    INFO] - loss: 0.16662848, learning_rate: 5.444e-05, global_step: 51, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.0022, interval_samples_per_second: 0.3153, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0326
+
+TrainProcess:  52%|███��█▏    | 52/100 [5:52:23<5:25:14, 406.54s/it][2025-11-01 08:58:17,220] [    INFO] - loss: 0.21518505, learning_rate: 5.333e-05, global_step: 52, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.6321, interval_samples_per_second: 0.3156, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0333
+
+TrainProcess:  53%|█████▎    | 53/100 [5:59:12<5:19:04, 407.34s/it][2025-11-01 09:05:06,410] [    INFO] - loss: 0.11518776, learning_rate: 5.222e-05, global_step: 53, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 409.1872, interval_samples_per_second: 0.3128, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0339
+
+TrainProcess:  54%|█████▍    | 54/100 [6:05:58<5:12:09, 407.16s/it][2025-11-01 09:11:53,150] [    INFO] - loss: 0.19170398, learning_rate: 5.111e-05, global_step: 54, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.7699, interval_samples_per_second: 0.3147, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0346
+
+TrainProcess:  55%|█████▌    | 55/100 [6:12:45<5:05:10, 406.90s/it][2025-11-01 09:18:39,419] [    INFO] - loss: 0.14399374, learning_rate: 5e-05, global_step: 55, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.2924, interval_samples_per_second: 0.315, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0352
+
+TrainProcess:  56%|█████▌    | 56/100 [6:19:33<4:58:36, 407.20s/it][2025-11-01 09:25:27,386] [    INFO] - loss: 0.27147686, learning_rate: 4.889e-05, global_step: 56, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.8786, interval_samples_per_second: 0.3138, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0358
+
+TrainProcess:  57%|█████▋    | 57/100 [6:26:21<4:52:02, 407.51s/it][2025-11-01 09:32:15,569] [    INFO] - loss: 0.18527454, learning_rate: 4.778e-05, global_step: 57, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.2632, interval_samples_per_second: 0.3135, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0365
+
+TrainProcess:  58%|█████▊    | 58/100 [6:33:06<4:44:45, 406.80s/it][2025-11-01 09:39:00,671] [    INFO] - loss: 0.1017971, learning_rate: 4.667e-05, global_step: 58, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.1291, interval_samples_per_second: 0.3159, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0371
+
+TrainProcess:  59%|█████▉    | 59/100 [6:39:51<4:37:34, 406.22s/it][2025-11-01 09:45:45,647] [    INFO] - loss: 0.05975801, learning_rate: 4.556e-05, global_step: 59, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.8463, interval_samples_per_second: 0.3162, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0378
+
+TrainProcess:  60%|██████    | 60/100 [6:46:37<4:30:46, 406.15s/it][2025-11-01 09:52:31,558] [    INFO] - loss: 0.23197478, learning_rate: 4.444e-05, global_step: 60, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.9956, interval_samples_per_second: 0.3153, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0384
+
+TrainProcess:  61%|██████    | 61/100 [6:53:25<4:24:26, 406.84s/it][2025-11-01 09:59:19,999] [    INFO] - loss: 0.08904499, learning_rate: 4.333e-05, global_step: 61, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.4414, interval_samples_per_second: 0.3134, interval_steps_per_second: 0.0024, progress_or_epoch: 0.039
+
+TrainProcess:  62%|██████▏   | 62/100 [7:00:12<4:17:37, 406.77s/it][2025-11-01 10:06:06,634] [    INFO] - loss: 0.14769191, learning_rate: 4.222e-05, global_step: 62, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.6277, interval_samples_per_second: 0.3148, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0397
+
+TrainProcess:  63%|██████▎   | 63/100 [7:06:58<4:10:44, 406.60s/it][2025-11-01 10:12:52,829] [    INFO] - loss: 0.07841766, learning_rate: 4.111e-05, global_step: 63, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.1816, interval_samples_per_second: 0.3151, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0403
+
+TrainProcess:  64%|██████▍   | 64/100 [7:13:45<4:03:57, 406.59s/it][2025-11-01 10:19:39,351] [    INFO] - loss: 0.0982312, learning_rate: 4e-05, global_step: 64, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.5782, interval_samples_per_second: 0.3148, interval_steps_per_second: 0.0025, progress_or_epoch: 0.041
+
+TrainProcess:  65%|██████▌   | 65/100 [7:20:33<3:57:30, 407.17s/it][2025-11-01 10:26:27,914] [    INFO] - loss: 0.14100146, learning_rate: 3.889e-05, global_step: 65, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.5058, interval_samples_per_second: 0.3133, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0416
+
+TrainProcess:  66%|██████▌   | 66/100 [7:27:21<3:50:48, 407.30s/it][2025-11-01 10:33:15,505] [    INFO] - loss: 0.28269625, learning_rate: 3.778e-05, global_step: 66, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.6186, interval_samples_per_second: 0.314, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0422
+
+TrainProcess:  67%|██████▋   | 67/100 [7:34:07<3:43:50, 407.00s/it][2025-11-01 10:40:01,841] [    INFO] - loss: 0.29586047, learning_rate: 3.667e-05, global_step: 67, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.2809, interval_samples_per_second: 0.3151, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0429
+
+TrainProcess:  68%|██████▊   | 68/100 [7:40:53<3:36:56, 406.75s/it][2025-11-01 10:46:47,961] [    INFO] - loss: 0.18546623, learning_rate: 3.556e-05, global_step: 68, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.1829, interval_samples_per_second: 0.3151, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0435
+
+TrainProcess:  69%|██████▉   | 69/100 [7:47:41<3:30:17, 407.03s/it][2025-11-01 10:53:35,657] [    INFO] - loss: 0.28093511, learning_rate: 3.444e-05, global_step: 69, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.6722, interval_samples_per_second: 0.314, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0442
+
+TrainProcess:  70%|███████   | 70/100 [7:54:30<3:23:47, 407.59s/it][2025-11-01 11:00:24,507] [    INFO] - loss: 0.15242034, learning_rate: 3.333e-05, global_step: 70, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.9017, interval_samples_per_second: 0.313, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0448
+
+TrainProcess:  71%|███████   | 71/100 [8:01:16<3:16:47, 407.15s/it][2025-11-01 11:07:10,663] [    INFO] - loss: 0.09792447, learning_rate: 3.222e-05, global_step: 71, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.1241, interval_samples_per_second: 0.3152, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0454
+
+TrainProcess:  72%|███████▏  | 72/100 [8:08:02<3:09:49, 406.76s/it][2025-11-01 11:13:56,507] [    INFO] - loss: 0.26866454, learning_rate: 3.111e-05, global_step: 72, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.8375, interval_samples_per_second: 0.3154, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0461
+
+TrainProcess:  73%|███████▎  | 73/100 [8:14:47<3:02:52, 406.38s/it][2025-11-01 11:20:42,054] [    INFO] - loss: 0.06001234, learning_rate: 3e-05, global_step: 73, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.4907, interval_samples_per_second: 0.3157, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0467
+
+TrainProcess:  74%|███████▍  | 74/100 [8:21:34<2:56:05, 406.37s/it][2025-11-01 11:27:28,423] [    INFO] - loss: 0.14044982, learning_rate: 2.889e-05, global_step: 74, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.3822, interval_samples_per_second: 0.315, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0474
+
+TrainProcess:  75%|███████▌  | 75/100 [8:28:22<2:49:36, 407.06s/it][2025-11-01 11:34:17,090] [    INFO] - loss: 0.17058212, learning_rate: 2.778e-05, global_step: 75, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.6496, interval_samples_per_second: 0.3132, interval_steps_per_second: 0.0024, progress_or_epoch: 0.048
+
+TrainProcess:  76%|███████▌  | 76/100 [8:35:08<2:42:38, 406.62s/it][2025-11-01 11:41:02,623] [    INFO] - loss: 0.15776825, learning_rate: 2.667e-05, global_step: 76, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.5977, interval_samples_per_second: 0.3156, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0486
+
+TrainProcess:  77%|███████▋  | 77/100 [8:41:53<2:35:43, 406.24s/it][2025-11-01 11:47:47,931] [    INFO] - loss: 0.12274396, learning_rate: 2.556e-05, global_step: 77, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.3328, interval_samples_per_second: 0.3158, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0493
+
+TrainProcess:  78%|███████▊  | 78/100 [8:48:38<2:28:50, 405.93s/it][2025-11-01 11:54:33,189] [    INFO] - loss: 0.07647455, learning_rate: 2.444e-05, global_step: 78, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.21, interval_samples_per_second: 0.3159, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0499
+
+TrainProcess:  79%|███████▉  | 79/100 [8:55:25<2:22:07, 406.06s/it][2025-11-01 12:01:19,539] [    INFO] - loss: 0.24748832, learning_rate: 2.333e-05, global_step: 79, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.3784, interval_samples_per_second: 0.315, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0506
+
+TrainProcess:  80%|████████  | 80/100 [9:02:13<2:15:34, 406.74s/it][2025-11-01 12:08:07,932] [    INFO] - loss: 0.24539894, learning_rate: 2.222e-05, global_step: 80, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.3534, interval_samples_per_second: 0.3135, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0512
+
+TrainProcess:  81%|████████  | 81/100 [9:08:59<2:08:44, 406.55s/it][2025-11-01 12:14:53,947] [    INFO] - loss: 0.11816031, learning_rate: 2.111e-05, global_step: 81, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.0794, interval_samples_per_second: 0.3152, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0518
+
+TrainProcess:  82%|████████▏ | 82/100 [9:15:44<2:01:50, 406.13s/it][2025-11-01 12:21:39,118] [    INFO] - loss: 0.12493551, learning_rate: 2e-05, global_step: 82, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.1452, interval_samples_per_second: 0.3159, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0525
+
+TrainProcess:  83%|████████▎ | 83/100 [9:22:30<1:55:01, 405.97s/it][2025-11-01 12:28:24,814] [    INFO] - loss: 0.07500303, learning_rate: 1.889e-05, global_step: 83, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.6077, interval_samples_per_second: 0.3156, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0531
+
+TrainProcess:  84%|████████▍ | 84/100 [9:29:15<1:48:12, 405.78s/it][2025-11-01 12:35:10,015] [    INFO] - loss: 0.08597636, learning_rate: 1.778e-05, global_step: 84, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.3177, interval_samples_per_second: 0.3158, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0538
+
+TrainProcess:  85%|████████▌ | 85/100 [9:36:03<1:41:34, 406.30s/it][2025-11-01 12:41:57,667] [    INFO] - loss: 0.16892493, learning_rate: 1.667e-05, global_step: 85, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.5144, interval_samples_per_second: 0.3141, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0544
+
+TrainProcess:  86%|████████▌ | 86/100 [9:42:49<1:34:47, 406.22s/it][2025-11-01 12:48:43,604] [    INFO] - loss: 0.1329937, learning_rate: 1.556e-05, global_step: 86, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.0372, interval_samples_per_second: 0.3152, interval_steps_per_second: 0.0025, progress_or_epoch: 0.055
+
+TrainProcess:  87%|████████▋ | 87/100 [9:49:33<1:27:54, 405.71s/it][2025-11-01 12:55:28,121] [    INFO] - loss: 0.1806075, learning_rate: 1.444e-05, global_step: 87, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.5198, interval_samples_per_second: 0.3164, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0557
+
+TrainProcess:  88%|████████▊ | 88/100 [9:56:19<1:21:06, 405.52s/it][2025-11-01 13:02:13,254] [    INFO] - loss: 0.14391142, learning_rate: 1.333e-05, global_step: 88, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.076, interval_samples_per_second: 0.316, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0563
+
+TrainProcess:  89%|████████▉ | 89/100 [10:03:04<1:14:19, 405.43s/it][2025-11-01 13:08:58,498] [    INFO] - loss: 0.25893754, learning_rate: 1.222e-05, global_step: 89, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.233, interval_samples_per_second: 0.3159, interval_steps_per_second: 0.0025, progress_or_epoch: 0.057
+
+TrainProcess:  90%|█████████ | 90/100 [10:09:50<1:07:35, 405.56s/it][2025-11-01 13:15:44,303] [    INFO] - loss: 0.20979571, learning_rate: 1.111e-05, global_step: 90, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.8473, interval_samples_per_second: 0.3154, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0576
+
+TrainProcess:  91%|█████████ | 91/100 [10:16:36<1:00:53, 405.93s/it][2025-11-01 13:22:31,067] [    INFO] - loss: 0.27406305, learning_rate: 1e-05, global_step: 91, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.7958, interval_samples_per_second: 0.3147, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0582
+
+TrainProcess:  92%|█████████▏| 92/100 [10:23:24<54:11, 406.46s/it]  [2025-11-01 13:29:18,826] [    INFO] - loss: 0.19165272, learning_rate: 8.889e-06, global_step: 92, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 407.7117, interval_samples_per_second: 0.3139, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0589
+
+TrainProcess:  93%|█████████▎| 93/100 [10:30:10<47:23, 406.21s/it][2025-11-01 13:36:04,427] [    INFO] - loss: 0.20233607, learning_rate: 7.778e-06, global_step: 93, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.6024, interval_samples_per_second: 0.3156, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0595
+
+TrainProcess:  94%|█████████▍| 94/100 [10:36:56<40:38, 406.34s/it][2025-11-01 13:42:51,032] [    INFO] - loss: 0.1774289, learning_rate: 6.667e-06, global_step: 94, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.6665, interval_samples_per_second: 0.3148, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0602
+
+TrainProcess:  95%|█████████▌| 95/100 [10:43:42<33:50, 406.11s/it][2025-11-01 13:49:36,663] [    INFO] - loss: 0.12968969, learning_rate: 5.556e-06, global_step: 95, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.5759, interval_samples_per_second: 0.3156, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0608
+
+TrainProcess:  96%|█████████▌| 96/100 [10:50:27<27:03, 405.84s/it][2025-11-01 13:56:21,893] [    INFO] - loss: 0.17552984, learning_rate: 4.444e-06, global_step: 96, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 405.2193, interval_samples_per_second: 0.3159, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0614
+
+TrainProcess:  97%|█████████▋| 97/100 [10:57:14<20:18, 406.06s/it][2025-11-01 14:03:08,438] [    INFO] - loss: 0.19446421, learning_rate: 3.333e-06, global_step: 97, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.5842, interval_samples_per_second: 0.3148, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0621
+
+TrainProcess:  98%|█████████▊| 98/100 [11:04:03<13:33, 406.95s/it][2025-11-01 14:09:57,441] [    INFO] - loss: 0.18020761, learning_rate: 2.222e-06, global_step: 98, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 408.9929, interval_samples_per_second: 0.313, interval_steps_per_second: 0.0024, progress_or_epoch: 0.0627
+
+TrainProcess:  99%|█████████▉| 99/100 [11:10:49<06:46, 406.76s/it][2025-11-01 14:16:43,792] [    INFO] - loss: 0.17214525, learning_rate: 1.111e-06, global_step: 99, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 406.3251, interval_samples_per_second: 0.315, interval_steps_per_second: 0.0025, progress_or_epoch: 0.0634
+
+TrainProcess: 100%|██████████| 100/100 [11:17:34<00:00, 406.23s/it][2025-11-01 14:23:28,773] [    INFO] - loss: 0.19793379, learning_rate: 0.0, global_step: 100, current_memory_allocated: 12.27591323852539, current_memory_reserved: 70.4673011302948, max_memory_allocated: 63.21619176864624, max_memory_reserved: 70.4673011302948, interval_runtime: 404.9897, interval_samples_per_second: 0.3161, interval_steps_per_second: 0.0025, progress_or_epoch: 0.064
+[2025-11-01 14:23:28,773] [    INFO] - 
+Training completed. 
+
+[2025-11-01 14:23:28,786] [    INFO] - train_runtime: 40654.6818, train_samples_per_second: 0.3148, train_steps_per_second: 0.0025, train_loss: 0.1801042765378952, progress_or_epoch: 0.064
+
+TrainProcess: 100%|██████████| 100/100 [11:17:34<00:00, 406.55s/it]
+[2025-11-01 14:23:28,791] [    INFO] - Saving model checkpoint to tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16
+[2025-11-01 14:23:28,884] [    INFO] - tokenizer config file saved in tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/tokenizer_config.json
+[2025-11-01 14:23:28,889] [    INFO] - Special tokens file saved in tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/special_tokens_map.json
+[2025-11-01 14:23:30,218] [    INFO] - Configuration saved in tmp_train_mmarco.jsonl/repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/config.json
+[2025-11-01 14:23:30,218] [    INFO] - ***** train metrics *****
+[2025-11-01 14:23:30,220] [    INFO] -   progress_or_epoch        =       0.064
+[2025-11-01 14:23:30,223] [    INFO] -   train_loss               =      0.1801
+[2025-11-01 14:23:30,223] [    INFO] -   train_runtime            = 11:17:34.68
+[2025-11-01 14:23:30,225] [    INFO] -   train_samples_per_second =      0.3148
+[2025-11-01 14:23:30,226] [    INFO] -   train_steps_per_second   =      0.0025
+sys:1: DeprecationWarning: builtin type swigvarlink has no __module__ attribute
+LAUNCH INFO 2025-11-01 14:23:39,426 Pod completed
+LAUNCH INFO 2025-11-01 14:23:39,427 Exit code 0

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/train_results.json

@@ -0,0 +1,7 @@
+{
+    "progress_or_epoch": 0.064,
+    "train_loss": 0.1801042765378952,
+    "train_runtime": 40654.6818,
+    "train_samples_per_second": 0.3148,
+    "train_steps_per_second": 0.0025
+}

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/trainer_state.json

@@ -0,0 +1,15 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "consumed_samples": 12800,
+  "epoch": 0.06399872002559949,
+  "global_step": 100,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [],
+  "max_steps": 100,
+  "num_train_epochs": 1,
+  "total_flos": 0,
+  "trial_name": null,
+  "trial_params": null
+}

repllama-v1-7b-lora-passage-merge-then-to-paddle-float16/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d76bf41c8e1c7f1d60199838f841298710c5c625751ed24f110c762c9c18adee
+size 3858