Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+training_log.txt filter=lfs diff=lfs merge=lfs -text
+wandb/run-20260225_055938-u2vegsv1/run-u2vegsv1.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20260225_063717-0ub00jhc/run-0ub00jhc.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20260225_094307-hmhb8ltr/run-hmhb8ltr.wandb filter=lfs diff=lfs merge=lfs -text
+wandb/run-20260225_111518-pg2w7c3p/run-pg2w7c3p.wandb filter=lfs diff=lfs merge=lfs -text

added_tokens.json

@@ -0,0 +1,36 @@
+{
+  "</box>": 151673,
+  "</img>": 151666,
+  "</quad>": 151669,
+  "</ref>": 151671,
+  "</tool_call>": 151658,
+  "<IMG_CONTEXT>": 151667,
+  "<box>": 151672,
+  "<cand>": 151674,
+  "<e_cand>": 151676,
+  "<e_s>": 151675,
+  "<img>": 151665,
+  "<quad>": 151668,
+  "<ref>": 151670,
+  "<tool_call>": 151657,
+  "<|box_end|>": 151649,
+  "<|box_start|>": 151648,
+  "<|endoftext|>": 151643,
+  "<|file_sep|>": 151664,
+  "<|fim_middle|>": 151660,
+  "<|fim_pad|>": 151662,
+  "<|fim_prefix|>": 151659,
+  "<|fim_suffix|>": 151661,
+  "<|im_end|>": 151645,
+  "<|im_start|>": 151644,
+  "<|image_pad|>": 151655,
+  "<|object_ref_end|>": 151647,
+  "<|object_ref_start|>": 151646,
+  "<|quad_end|>": 151651,
+  "<|quad_start|>": 151650,
+  "<|repo_name|>": 151663,
+  "<|video_pad|>": 151656,
+  "<|vision_end|>": 151653,
+  "<|vision_pad|>": 151654,
+  "<|vision_start|>": 151652
+}

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.3191605252492171,
+    "train_runtime": 117088.0105,
+    "train_samples": -1,
+    "train_samples_per_second": 0.318,
+    "train_steps_per_second": 0.079
+}

config.json

@@ -0,0 +1,237 @@
+{
+  "_commit_hash": null,
+  "_name_or_path": "../pretrained/InternVL3-2B",
+  "architectures": [
+    "InternVLChatModel"
+  ],
+  "auto_map": {
+    "AutoConfig": "configuration_internvl_chat.InternVLChatConfig",
+    "AutoModel": "modeling_internvl_chat.InternVLChatModel",
+    "AutoModelForCausalLM": "modeling_internvl_chat.InternVLChatModel"
+  },
+  "bev_image_size": 448,
+  "downsample_ratio": 0.5,
+  "dual_text_pos_injection": true,
+  "dynamic_image_size": false,
+  "force_image_size": 448,
+  "hidden_size": 1536,
+  "image_fold": null,
+  "llm_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "./pretrained/Qwen2.5-32B-Instruct",
+    "add_cross_attention": false,
+    "architectures": [
+      "Qwen2ForCausalLM"
+    ],
+    "attention_dropout": 0.0,
+    "attn_implementation": "flash_attention_2",
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": 151643,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": 151643,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "initializer_range": 0.02,
+    "intermediate_size": 8960,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 70,
+    "min_length": 0,
+    "model_type": "qwen2",
+    "moe_config": null,
+    "no_repeat_ngram_size": 0,
+    "num_attention_heads": 12,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 2,
+    "num_return_sequences": 1,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": {
+      "factor": 2.0,
+      "rope_type": "dynamic",
+      "type": "dynamic"
+    },
+    "rope_theta": 1000000.0,
+    "sep_token_id": null,
+    "sliding_window": null,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": false,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers_version": "4.37.2",
+    "typical_p": 1.0,
+    "use_bfloat16": true,
+    "use_cache": false,
+    "use_sliding_window": false,
+    "vocab_size": 151677
+  },
+  "max_dynamic_patch": 12,
+  "min_dynamic_patch": 1,
+  "model_type": "internvl_chat",
+  "num_image_token_bev": 256,
+  "num_image_token_ego": 32,
+  "pad2square": false,
+  "ps_version": "v2",
+  "select_layer": -1,
+  "system_message": "You are an autonomous navigation agent operating in indoor environments. You receive spatial information through position embeddings injected into visual features and text tokens. Use the BEV map, position embeddings, and semantic information to make navigation decisions. When the target object is detected (<target> marker), navigate directly to it. Otherwise, explore frontiers strategically to find the goal object.",
+  "template": "internvl2_5_nav",
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": null,
+  "use_backbone_lora": 0,
+  "use_llm_lora": 64,
+  "use_pairwise_spatial_encoder": false,
+  "use_position_embeddings": true,
+  "use_thumbnail": true,
+  "vision_config": {
+    "_attn_implementation_autoset": true,
+    "_name_or_path": "OpenGVLab/InternViT-6B-448px-V1-5",
+    "add_cross_attention": false,
+    "architectures": [
+      "InternVisionModel"
+    ],
+    "attention_dropout": 0.0,
+    "auto_map": {
+      "AutoConfig": "configuration_intern_vit.InternVisionConfig",
+      "AutoModel": "modeling_intern_vit.InternVisionModel"
+    },
+    "bad_words_ids": null,
+    "begin_suppress_tokens": null,
+    "bos_token_id": null,
+    "capacity_factor": 1.2,
+    "chunk_size_feed_forward": 0,
+    "cross_attention_hidden_size": null,
+    "decoder_start_token_id": null,
+    "diversity_penalty": 0.0,
+    "do_sample": false,
+    "drop_path_rate": 0.0,
+    "dropout": 0.0,
+    "early_stopping": false,
+    "encoder_no_repeat_ngram_size": 0,
+    "eos_token_id": null,
+    "eval_capacity_factor": 1.4,
+    "exponential_decay_length_penalty": null,
+    "finetuning_task": null,
+    "forced_bos_token_id": null,
+    "forced_eos_token_id": null,
+    "hidden_act": "gelu",
+    "hidden_size": 1024,
+    "id2label": {
+      "0": "LABEL_0",
+      "1": "LABEL_1"
+    },
+    "image_size": 448,
+    "initializer_factor": 0.1,
+    "initializer_range": 1e-10,
+    "intermediate_size": 4096,
+    "is_decoder": false,
+    "is_encoder_decoder": false,
+    "label2id": {
+      "LABEL_0": 0,
+      "LABEL_1": 1
+    },
+    "laux_allreduce": "all_nodes",
+    "layer_norm_eps": 1e-06,
+    "length_penalty": 1.0,
+    "max_length": 20,
+    "min_length": 0,
+    "model_type": "intern_vit_6b",
+    "moe_coeff_ratio": 0.5,
+    "moe_intermediate_size": 768,
+    "moe_output_scale": 4.0,
+    "no_repeat_ngram_size": 0,
+    "noisy_gate_policy": "RSample_before",
+    "norm_type": "layer_norm",
+    "num_attention_heads": 16,
+    "num_beam_groups": 1,
+    "num_beams": 1,
+    "num_channels": 3,
+    "num_experts": 8,
+    "num_hidden_layers": 24,
+    "num_return_sequences": 1,
+    "num_routed_experts": 4,
+    "num_shared_experts": 4,
+    "output_attentions": false,
+    "output_hidden_states": false,
+    "output_scores": false,
+    "pad_token_id": null,
+    "patch_size": 14,
+    "prefix": null,
+    "problem_type": null,
+    "pruned_heads": {},
+    "qk_normalization": false,
+    "qkv_bias": true,
+    "remove_invalid_values": false,
+    "repetition_penalty": 1.0,
+    "return_dict": true,
+    "return_dict_in_generate": false,
+    "sep_token_id": null,
+    "shared_expert_intermediate_size": 3072,
+    "suppress_tokens": null,
+    "task_specific_params": null,
+    "temperature": 1.0,
+    "tf_legacy_loss": false,
+    "tie_encoder_decoder": false,
+    "tie_word_embeddings": true,
+    "tokenizer_class": null,
+    "top_k": 50,
+    "top_p": 1.0,
+    "torch_dtype": "bfloat16",
+    "torchscript": false,
+    "transformers_version": "4.37.2",
+    "typical_p": 1.0,
+    "use_bfloat16": true,
+    "use_flash_attn": true,
+    "use_moe": false,
+    "use_residual": true,
+    "use_rts": false,
+    "use_weighted_residual": false
+  },
+  "vit_bev_freeze": true,
+  "vit_bev_lora_rank": 64,
+  "vit_bev_use_lora": true,
+  "vit_rgb_freeze": true,
+  "vit_rgb_lora_rank": 16,
+  "vit_rgb_use_lora": true
+}

generation_config.json

@@ -0,0 +1,4 @@
+{
+  "_from_model_config": true,
+  "transformers_version": "4.37.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f8f7cace65c206ed677803c7a56351d148073ccde09768ac4e0c8013155650f
+size 4997765528

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df61243e02c9244bc214ea44c598360ae3a028126816819deb77d6ca8a3c3152
+size 8563840

resolved_train_config.json

@@ -0,0 +1,83 @@
+{
+  "model_name_or_path": "../pretrained/InternVL3-2B",
+  "freeze_backbone": true,
+  "unfreeze_vit_layers": 0,
+  "use_backbone_lora": 0,
+  "freeze_llm": true,
+  "unfreeze_lm_head": false,
+  "use_llm_lora": 64,
+  "freeze_mlp": false,
+  "vision_select_layer": -1,
+  "grad_checkpoint": true,
+  "gradient_checkpointing": true,
+  "drop_path_rate": 0.0,
+  "ps_version": "v2",
+  "use_fast_tokenizer": true,
+  "max_seq_length": 24576,
+  "force_image_size": 448,
+  "down_sample_ratio": 0.5,
+  "pad2square": true,
+  "conv_style": "internvl2_5_nav",
+  "meta_path": "",
+  "use_data_resampling": false,
+  "dynamic_image_size": false,
+  "use_thumbnail": true,
+  "min_dynamic_patch": 1,
+  "max_dynamic_patch": 12,
+  "normalize_type": "imagenet",
+  "use_packed_ds": true,
+  "num_images_expected": 1000,
+  "max_packed_tokens": 24576,
+  "max_buffer_size": 20,
+  "log_freq": 1000,
+  "strict_mode": false,
+  "replacement": true,
+  "allow_overflow": false,
+  "loss_reduction": "square",
+  "loss_reduction_all_gather": false,
+  "seed": 42,
+  "output_dir": "",
+  "overwrite_output_dir": true,
+  "report_to": "wandb",
+  "save_strategy": "steps",
+  "save_total_limit": 1,
+  "save_steps": 0.5,
+  "logging_steps": 1,
+  "evaluation_strategy": "no",
+  "dataloader_num_workers": 2,
+  "group_by_length": false,
+  "deepspeed": "zero_stage2_config_acc1.json",
+  "remove_unused_columns": false,
+  "do_train": true,
+  "bf16": true,
+  "learning_rate": 0.0001,
+  "weight_decay": 0.01,
+  "warmup_ratio": 0.03,
+  "lr_scheduler_type": "cosine",
+  "max_grad_norm": 1.0,
+  "num_train_epochs": 1,
+  "per_device_train_batch_size": 1,
+  "gradient_accumulation_steps": 1,
+  "max_steps": 8000,
+  "template_name": "BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY",
+  "use_nav_system_message": true,
+  "use_position_embeddings": true,
+  "use_pairwise_spatial_encoder": false,
+  "use_dual_vit": true,
+  "bev_image_size": 448,
+  "num_image_token_bev": 256,
+  "num_image_token_ego": 32,
+  "vit_bev_freeze": true,
+  "vit_bev_use_lora": true,
+  "vit_bev_lora_rank": 64,
+  "vit_rgb_freeze": true,
+  "vit_rgb_use_lora": true,
+  "vit_rgb_lora_rank": 16,
+  "position_placeholders": [
+    "<s>",
+    "<cand>",
+    "<e_s>",
+    "<e_cand>"
+  ],
+  "dual_text_pos_injection": true
+}

special_tokens_map.json

@@ -0,0 +1,31 @@
+{
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "eos_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer_config.json

@@ -0,0 +1,313 @@
+{
+  "add_bos_token": false,
+  "add_eos_token": false,
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "128245": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151643": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151644": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151645": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151646": {
+      "content": "<|object_ref_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151647": {
+      "content": "<|object_ref_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151648": {
+      "content": "<|box_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151649": {
+      "content": "<|box_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151650": {
+      "content": "<|quad_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151651": {
+      "content": "<|quad_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151652": {
+      "content": "<|vision_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151653": {
+      "content": "<|vision_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151654": {
+      "content": "<|vision_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151655": {
+      "content": "<|image_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151656": {
+      "content": "<|video_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151657": {
+      "content": "<tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151658": {
+      "content": "</tool_call>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151659": {
+      "content": "<|fim_prefix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151660": {
+      "content": "<|fim_middle|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151661": {
+      "content": "<|fim_suffix|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151662": {
+      "content": "<|fim_pad|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151663": {
+      "content": "<|repo_name|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151664": {
+      "content": "<|file_sep|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "151665": {
+      "content": "<img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151666": {
+      "content": "</img>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151667": {
+      "content": "<IMG_CONTEXT>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151668": {
+      "content": "<quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151669": {
+      "content": "</quad>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151670": {
+      "content": "<ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151671": {
+      "content": "</ref>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151672": {
+      "content": "<box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151673": {
+      "content": "</box>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151674": {
+      "content": "<cand>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151675": {
+      "content": "<e_s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151676": {
+      "content": "<e_cand>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|object_ref_start|>",
+    "<|object_ref_end|>",
+    "<|box_start|>",
+    "<|box_end|>",
+    "<|quad_start|>",
+    "<|quad_end|>",
+    "<|vision_start|>",
+    "<|vision_end|>",
+    "<|vision_pad|>",
+    "<|image_pad|>",
+    "<|video_pad|>"
+  ],
+  "bos_token": null,
+  "chat_template": "{%- if messages[0]['role'] == 'system' %}{{- '<|im_start|>system\n' + messages[0]['content'] + '<|im_end|>\n' }}{%- else %}{{- '<|im_start|>system\n你是书生·万象，英文名是InternVL，是由上海人工智能实验室、清华大学及多家合作单位联合开发的多模态大语言模型。<|im_end|>\n' }}{%- endif %}{% for message in messages %}{%- if messages[0]['role'] != 'system' or not loop.first %}{{'<|im_start|>' + message['role'] + '\n'}}{% if message['content'] is string %}{{ message['content'] }}{% else %}{% for content in message['content'] %}{% if content['type'] == 'image' %}{{ '<image>\n' }}{% elif content['type'] == 'video' %}{{ '<video>\n' }}{% elif content['type'] == 'text' %}{{ content['text'] }}{% endif %}{% endfor %}{% endif %}{{'<|im_end|>\n'}}{%- endif %}{% endfor %}{% if add_generation_prompt %}{{'<|im_start|>assistant\n' }}{% endif %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "errors": "replace",
+  "extra_special_tokens": {},
+  "model_max_length": 24576,
+  "pad_token": "<|endoftext|>",
+  "split_special_tokens": false,
+  "tokenizer_class": "Qwen2Tokenizer",
+  "unk_token": null
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 1.0,
+    "train_loss": 0.3191605252492171,
+    "train_runtime": 117088.0105,
+    "train_samples": -1,
+    "train_samples_per_second": 0.318,
+    "train_steps_per_second": 0.079
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e7589102ab8fa3bd0e675ac7553bdd0895800c35df3fa2763096b83dac317d48
+size 6328

training_log.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8060c9bf6ec381f46c7e1d689880c3c357d7891835ec14e0468f2ba7af26de61
+size 15676128

wandb/debug-internal.log

@@ -0,0 +1,29 @@
+{"time":"2026-02-25T11:15:18.266363836+08:00","level":"INFO","msg":"stream: starting","core version":"0.22.3"}
+{"time":"2026-02-25T11:15:23.025454392+08:00","level":"INFO","msg":"stream: created new stream","id":"pg2w7c3p"}
+{"time":"2026-02-25T11:15:23.025989355+08:00","level":"INFO","msg":"handler: started","stream_id":"pg2w7c3p"}
+{"time":"2026-02-25T11:15:23.029061332+08:00","level":"INFO","msg":"stream: started","id":"pg2w7c3p"}
+{"time":"2026-02-25T11:15:23.029075662+08:00","level":"INFO","msg":"sender: started","stream_id":"pg2w7c3p"}
+{"time":"2026-02-25T11:15:23.02907461+08:00","level":"INFO","msg":"writer: started","stream_id":"pg2w7c3p"}
+{"time":"2026-02-25T15:48:54.519398391+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/tsaisplus-nanyang-technological-university-singapore/prompt_revision/pg2w7c3p/file_stream\": EOF"}
+{"time":"2026-02-25T21:07:59.713263866+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": unexpected EOF"}
+{"time":"2026-02-26T01:46:00.156791646+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":64931}
+{"time":"2026-02-26T01:46:01.26570749+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T01:46:04.155088853+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T02:27:44.815648016+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":68065}
+{"time":"2026-02-26T02:27:46.348757133+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T02:27:49.049539083+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T03:22:31.469120504+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T03:22:33.406774848+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":72171}
+{"time":"2026-02-26T03:22:34.088783654+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T08:26:46.733939909+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T08:26:53.80398635+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":95045}
+{"time":"2026-02-26T08:26:57.006489641+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T09:11:42.493210656+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/tsaisplus-nanyang-technological-university-singapore/prompt_revision/pg2w7c3p/file_stream\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T12:21:30.540150918+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":112659}
+{"time":"2026-02-26T12:21:31.920394554+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T12:21:34.277988733+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T19:46:56.115836343+08:00","level":"INFO","msg":"stream: closing","id":"pg2w7c3p"}
+{"time":"2026-02-26T19:47:01.871500678+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-02-26T19:47:02.310370252+08:00","level":"INFO","msg":"handler: closed","stream_id":"pg2w7c3p"}
+{"time":"2026-02-26T19:47:02.312032251+08:00","level":"INFO","msg":"sender: closed","stream_id":"pg2w7c3p"}
+{"time":"2026-02-26T19:47:02.312485027+08:00","level":"INFO","msg":"stream: closed","id":"pg2w7c3p"}

wandb/debug.log

@@ -0,0 +1,24 @@
+2026-02-25 11:15:18,028 INFO    MainThread:121696 [wandb_setup.py:_flush():81] Current SDK version is 0.22.3
+2026-02-25 11:15:18,030 INFO    MainThread:121696 [wandb_setup.py:_flush():81] Configure stats pid to 121696
+2026-02-25 11:15:18,031 INFO    MainThread:121696 [wandb_setup.py:_flush():81] Loading settings from /mnt/petrelfs/wangmaonan/.config/wandb/settings
+2026-02-25 11:15:18,031 INFO    MainThread:121696 [wandb_setup.py:_flush():81] Loading settings from /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/wandb/settings
+2026-02-25 11:15:18,032 INFO    MainThread:121696 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-02-25 11:15:18,032 INFO    MainThread:121696 [wandb_init.py:setup_run_log_directory():706] Logging user logs to /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/wandb/run-20260225_111518-pg2w7c3p/logs/debug.log
+2026-02-25 11:15:18,033 INFO    MainThread:121696 [wandb_init.py:setup_run_log_directory():707] Logging internal logs to /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/wandb/run-20260225_111518-pg2w7c3p/logs/debug-internal.log
+2026-02-25 11:15:18,034 INFO    MainThread:121696 [wandb_init.py:init():833] calling init triggers
+2026-02-25 11:15:18,034 INFO    MainThread:121696 [wandb_init.py:init():838] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-02-25 11:15:18,035 INFO    MainThread:121696 [wandb_init.py:init():881] starting backend
+2026-02-25 11:15:18,255 INFO    MainThread:121696 [wandb_init.py:init():884] sending inform_init request
+2026-02-25 11:15:18,261 INFO    MainThread:121696 [wandb_init.py:init():892] backend started and connected
+2026-02-25 11:15:18,263 INFO    MainThread:121696 [wandb_init.py:init():962] updated telemetry
+2026-02-25 11:15:18,290 INFO    MainThread:121696 [wandb_init.py:init():986] communicating run to backend with 90.0 second timeout
+2026-02-25 11:15:28,713 INFO    MainThread:121696 [wandb_init.py:init():1033] starting run threads in backend
+2026-02-25 11:15:28,992 INFO    MainThread:121696 [wandb_run.py:_console_start():2506] atexit reg
+2026-02-25 11:15:28,993 INFO    MainThread:121696 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2026-02-25 11:15:28,993 INFO    MainThread:121696 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2026-02-25 11:15:28,994 INFO    MainThread:121696 [wandb_run.py:_redirect():2446] Redirects installed.
+2026-02-25 11:15:28,999 INFO    MainThread:121696 [wandb_init.py:init():1073] run started, returning control to user process
+2026-02-25 11:15:29,002 INFO    MainThread:121696 [wandb_run.py:_config_callback():1390] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'torch.bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['InternVLChatModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '../pretrained/InternVL3-2B', '_commit_hash': None, '_attn_implementation_internal': None, 'transformers_version': None, 'auto_map': {'AutoConfig': 'configuration_internvl_chat.InternVLChatConfig', 'AutoModel': 'modeling_internvl_chat.InternVLChatModel', 'AutoModelForCausalLM': 'modeling_internvl_chat.InternVLChatModel'}, 'hidden_size': 1536, 'image_fold': None, 'model_type': 'internvl_chat', 'system_message': 'You are an autonomous navigation agent operating in indoor environments. You receive spatial information through position embeddings injected into visual features and text tokens. Use the BEV map, position embeddings, and semantic information to make navigation decisions. When the target object is detected (<target> marker), navigate directly to it. Otherwise, explore frontiers strategically to find the goal object.', 'vision_config': {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': True, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['InternVisionModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'OpenGVLab/InternViT-6B-448px-V1-5', 'transformers_version': '4.37.2', '_attn_implementation_autoset': True, 'auto_map': {'AutoConfig': 'configuration_intern_vit.InternVisionConfig', 'AutoModel': 'modeling_intern_vit.InternVisionModel'}, 'capacity_factor': 1.2, 'eval_capacity_factor': 1.4, 'laux_allreduce': 'all_nodes', 'model_type': 'intern_vit_6b', 'moe_coeff_ratio': 0.5, 'moe_intermediate_size': 768, 'moe_output_scale': 4.0, 'noisy_gate_policy': 'RSample_before', 'num_experts': 8, 'num_routed_experts': 4, 'num_shared_experts': 4, 'shared_expert_intermediate_size': 3072, 'use_moe': False, 'use_residual': True, 'use_rts': False, 'use_weighted_residual': False, 'hidden_size': 1024, 'intermediate_size': 4096, 'dropout': 0.0, 'drop_path_rate': 0.0, 'num_hidden_layers': 24, 'num_attention_heads': 16, 'num_channels': 3, 'patch_size': 14, 'image_size': 448, 'initializer_range': 1e-10, 'initializer_factor': 0.1, 'attention_dropout': 0.0, 'layer_norm_eps': 1e-06, 'hidden_act': 'gelu', 'norm_type': 'layer_norm', 'qkv_bias': True, 'qk_normalization': False, 'use_flash_attn': True}, 'llm_config': {'vocab_size': 151677, 'max_position_embeddings': 32768, 'hidden_size': 1536, 'intermediate_size': 8960, 'num_hidden_layers': 28, 'num_attention_heads': 12, 'use_sliding_window': False, 'sliding_window': None, 'max_window_layers': 70, 'num_key_value_heads': 2, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 1000000.0, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': True, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['Qwen2ForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 151643, 'pad_token_id': None, 'eos_token_id': 151643, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': './pretrained/Qwen2.5-32B-Instruct', 'transformers_version': '4.37.2', '_attn_implementation_autoset': True, 'model_type': 'qwen2', 'moe_config': None, 'rope_scaling': {'factor': 2.0, 'rope_type': 'dynamic', 'type': 'dynamic'}, 'attn_implementation': 'flash_attention_2'}, 'use_backbone_lora': 0, 'use_llm_lora': 64, 'pad2square': False, 'select_layer': -1, 'force_image_size': 448, 'downsample_ratio': 0.5, 'template': 'internvl2_5_nav', 'dynamic_image_size': False, 'use_thumbnail': True, 'ps_version': 'v2', 'min_dynamic_patch': 1, 'max_dynamic_patch': 12, 'num_image_token_bev': 256, 'num_image_token_ego': 32, 'use_pairwise_spatial_encoder': False, 'use_position_embeddings': True, 'dual_text_pos_injection': True, 'bev_image_size': 448, 'vit_bev_freeze': True, 'vit_bev_use_lora': True, 'vit_bev_lora_rank': 64, 'vit_rgb_freeze': True, 'vit_rgb_use_lora': True, 'vit_rgb_lora_rank': 16, 'output_dir': '/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': 9300, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'runs/Feb25_11-14-26_SH-IDC1-10-140-37-90', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 0.5, 'save_total_limit': 1, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 2, 'past_index': -1, 'run_name': 'a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY_steps9300_gpus4_acc1', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'deepspeed': 'zero_stage2_config_acc1.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': False, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None}
+2026-02-26 19:46:56,115 INFO    wandb-AsyncioManager-main:121696 [service_client.py:_forward_responses():80] Reached EOF.
+2026-02-26 19:46:56,116 INFO    wandb-AsyncioManager-main:121696 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

wandb/run-20260225_055938-u2vegsv1/files/config.yaml

@@ -0,0 +1,767 @@
+_attn_implementation_internal:
+    value: null
+_commit_hash:
+    value: null
+_name_or_path:
+    value: ../pretrained/InternVL3-2B
+_wandb:
+    value:
+        cli_version: 0.22.3
+        e:
+            lcgegjlerikh15wodksbpfheadlvwekl:
+                args:
+                    - /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/resolved_train_config.json
+                codePath: InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py
+                codePathLocal: internvl_cleaned/train/internvl_chat_finetune.py
+                cpu_count: 64
+                cpu_count_logical: 128
+                cudaVersion: "12.2"
+                disk:
+                    /:
+                        total: "524945911808"
+                        used: "39460401152"
+                email: caiy0039@e.ntu.edu.sg
+                executable: /mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/bin/python3.10
+                git:
+                    commit: f7d6fbe6a8031d29a95c2f3be79e99f96670a12e
+                    remote: git@github.com:Yuxin916/CL_CoTNav.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-9e78ad4b-b304-a199-38f4-24b3acd9b531
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-19c9429b-5ecf-0b76-7d81-03c36a449f32
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-57520d78-d04a-3028-bd40-5ebd78e123e6
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-0d94270a-8ad3-b3a6-4acd-145fccc36d85
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-b9b6fe80-e37a-b622-0deb-27ef46a965ff
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-7b870038-dbe6-4039-b6bf-f90517f43f6c
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-e3065d01-9539-64c9-417f-91273b521051
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-28ce6eed-85b8-155a-5a5d-412dfb1dd1c0
+                host: SH-IDC1-10-140-37-45
+                memory:
+                    total: "1081627828224"
+                os: Linux-3.10.0-957.el7.x86_64-x86_64-with-glibc2.17
+                program: /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py
+                python: CPython 3.10.18
+                root: /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY
+                slurm:
+                    cluster_name: cluster_sproject3
+                    conf: /etc/slurm/slurm.conf
+                    cpus_on_node: "128"
+                    cpus_per_task: "8"
+                    distribution: cyclic
+                    gtids: "0"
+                    job_account: research
+                    job_cpus_per_node: "128"
+                    job_cpus_per_node_pack_group_0: "128"
+                    job_gid: "200000139"
+                    job_gpus: 0,1,2,3,4,5,6,7
+                    job_id: "7464467"
+                    job_name: vlm_ft
+                    job_nodelist: SH-IDC1-10-140-37-45
+                    job_num_nodes: "1"
+                    job_partition: interntmp
+                    job_qos: normal
+                    job_uid: "200000139"
+                    job_user: wangmaonan
+                    jobid: "7464467"
+                    launch_node_ipaddr: 10.140.37.45
+                    localid: "0"
+                    mem_per_node: "49152"
+                    nnodes: "1"
+                    node_aliases: (null)
+                    nodeid: "0"
+                    nodelist: SH-IDC1-10-140-37-45
+                    nprocs: "1"
+                    ntasks: "1"
+                    ntasks_per_node: "1"
+                    prio_process: "0"
+                    procid: "0"
+                    srun_comm_host: 10.140.37.45
+                    srun_comm_port: "44377"
+                    step_gpus: 0,1,2,3
+                    step_id: "0"
+                    step_launcher_port: "44377"
+                    step_nodelist: SH-IDC1-10-140-37-45
+                    step_num_nodes: "1"
+                    step_num_tasks: "1"
+                    step_tasks_per_node: "1"
+                    stepid: "0"
+                    submit_dir: /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav
+                    submit_host: SH-IDC1-10-140-37-45
+                    task_pid: "151799"
+                    tasks_per_node: "1"
+                    topology_addr: SH-IDC1-10-140-37-45
+                    topology_addr_pattern: node
+                    umask: "0002"
+                    working_cluster: cluster_sproject3:SH-IDC1-10-140-37-161:6817:9216:109
+                startedAt: "2026-02-24T21:59:38.884470Z"
+                writerId: lcgegjlerikh15wodksbpfheadlvwekl
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 63
+                - 71
+                - 98
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 63
+                - 71
+                - 98
+                - 105
+            "3":
+                - 7
+                - 13
+                - 66
+            "4": 3.10.18
+            "5": 0.22.3
+            "6": 4.37.2
+            "9":
+                "1": transformers_trainer
+            "12": 0.22.3
+            "13": linux-x86_64
+adafactor:
+    value: false
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+add_cross_attention:
+    value: false
+architectures:
+    value:
+        - InternVLChatModel
+auto_find_batch_size:
+    value: false
+auto_map:
+    value:
+        AutoConfig: configuration_internvl_chat.InternVLChatConfig
+        AutoModel: modeling_internvl_chat.InternVLChatModel
+        AutoModelForCausalLM: modeling_internvl_chat.InternVLChatModel
+bad_words_ids:
+    value: null
+begin_suppress_tokens:
+    value: null
+bev_image_size:
+    value: 448
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: null
+chunk_size_feed_forward:
+    value: 0
+cross_attention_hidden_size:
+    value: null
+data_seed:
+    value: null
+dataloader_drop_last:
+    value: false
+dataloader_num_workers:
+    value: 12
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+decoder_start_token_id:
+    value: null
+deepspeed:
+    value: zero_stage2_config_acc1.json
+disable_tqdm:
+    value: false
+dispatch_batches:
+    value: null
+diversity_penalty:
+    value: 0
+do_eval:
+    value: false
+do_predict:
+    value: false
+do_sample:
+    value: false
+do_train:
+    value: true
+downsample_ratio:
+    value: 0.5
+dual_text_pos_injection:
+    value: true
+dynamic_image_size:
+    value: false
+early_stopping:
+    value: false
+encoder_no_repeat_ngram_size:
+    value: 0
+eos_token_id:
+    value: null
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_steps:
+    value: null
+evaluation_strategy:
+    value: "no"
+exponential_decay_length_penalty:
+    value: null
+finetuning_task:
+    value: null
+force_image_size:
+    value: 448
+forced_bos_token_id:
+    value: null
+forced_eos_token_id:
+    value: null
+fp16:
+    value: false
+fp16_backend:
+    value: auto
+fp16_full_eval:
+    value: false
+fp16_opt_level:
+    value: O1
+fsdp:
+    value: []
+fsdp_config:
+    value:
+        min_num_params: 0
+        xla: false
+        xla_fsdp_grad_ckpt: false
+fsdp_min_num_params:
+    value: 0
+fsdp_transformer_layer_cls_to_wrap:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: true
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+group_by_length:
+    value: false
+half_precision_backend:
+    value: auto
+hidden_size:
+    value: 1536
+hub_always_push:
+    value: false
+hub_model_id:
+    value: null
+hub_private_repo:
+    value: false
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+image_fold:
+    value: null
+include_inputs_for_metrics:
+    value: false
+include_num_input_tokens_seen:
+    value: false
+include_tokens_per_second:
+    value: false
+is_decoder:
+    value: false
+is_encoder_decoder:
+    value: false
+jit_mode_eval:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.0001
+length_column_name:
+    value: length
+length_penalty:
+    value: 1
+llm_config:
+    value:
+        _attn_implementation_autoset: true
+        _name_or_path: ./pretrained/Qwen2.5-32B-Instruct
+        add_cross_attention: false
+        architectures:
+            - Qwen2ForCausalLM
+        attention_dropout: 0
+        attn_implementation: flash_attention_2
+        bad_words_ids: null
+        begin_suppress_tokens: null
+        bos_token_id: 151643
+        chunk_size_feed_forward: 0
+        cross_attention_hidden_size: null
+        decoder_start_token_id: null
+        diversity_penalty: 0
+        do_sample: false
+        early_stopping: false
+        encoder_no_repeat_ngram_size: 0
+        eos_token_id: 151643
+        exponential_decay_length_penalty: null
+        finetuning_task: null
+        forced_bos_token_id: null
+        forced_eos_token_id: null
+        hidden_act: silu
+        hidden_size: 1536
+        id2label:
+            "0": LABEL_0
+            "1": LABEL_1
+        initializer_range: 0.02
+        intermediate_size: 8960
+        is_decoder: false
+        is_encoder_decoder: false
+        label2id:
+            LABEL_0: 0
+            LABEL_1: 1
+        length_penalty: 1
+        max_length: 20
+        max_position_embeddings: 32768
+        max_window_layers: 70
+        min_length: 0
+        model_type: qwen2
+        moe_config: null
+        no_repeat_ngram_size: 0
+        num_attention_heads: 12
+        num_beam_groups: 1
+        num_beams: 1
+        num_hidden_layers: 28
+        num_key_value_heads: 2
+        num_return_sequences: 1
+        output_attentions: false
+        output_hidden_states: false
+        output_scores: false
+        pad_token_id: null
+        prefix: null
+        problem_type: null
+        remove_invalid_values: false
+        repetition_penalty: 1
+        return_dict: true
+        return_dict_in_generate: false
+        rms_norm_eps: 1e-06
+        rope_scaling:
+            factor: 2
+            rope_type: dynamic
+            type: dynamic
+        rope_theta: 1e+06
+        sep_token_id: null
+        sliding_window: null
+        suppress_tokens: null
+        task_specific_params: null
+        temperature: 1
+        tf_legacy_loss: false
+        tie_encoder_decoder: false
+        tie_word_embeddings: false
+        tokenizer_class: null
+        top_k: 50
+        top_p: 1
+        torch_dtype: bfloat16
+        torchscript: false
+        transformers_version: 4.37.2
+        typical_p: 1
+        use_bfloat16: true
+        use_cache: false
+        use_sliding_window: false
+        vocab_size: 151677
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: 0
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_dir:
+    value: runs/Feb25_05-58-51_SH-IDC1-10-140-37-45
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 1
+logging_strategy:
+    value: steps
+lr_scheduler_type:
+    value: cosine
+max_dynamic_patch:
+    value: 12
+max_grad_norm:
+    value: 1
+max_length:
+    value: 20
+max_steps:
+    value: 11000
+metric_for_best_model:
+    value: null
+min_dynamic_patch:
+    value: 1
+min_length:
+    value: 0
+model_type:
+    value: internvl_chat
+mp_parameters:
+    value: ""
+neftune_noise_alpha:
+    value: null
+no_cuda:
+    value: false
+no_repeat_ngram_size:
+    value: 0
+num_beam_groups:
+    value: 1
+num_beams:
+    value: 1
+num_image_token_bev:
+    value: 256
+num_image_token_ego:
+    value: 32
+num_return_sequences:
+    value: 1
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY
+output_hidden_states:
+    value: false
+output_scores:
+    value: false
+overwrite_output_dir:
+    value: true
+pad_token_id:
+    value: null
+pad2square:
+    value: false
+past_index:
+    value: -1
+per_device_eval_batch_size:
+    value: 8
+per_device_train_batch_size:
+    value: 1
+per_gpu_eval_batch_size:
+    value: null
+per_gpu_train_batch_size:
+    value: null
+prediction_loss_only:
+    value: false
+prefix:
+    value: null
+problem_type:
+    value: null
+ps_version:
+    value: v2
+push_to_hub:
+    value: false
+push_to_hub_model_id:
+    value: null
+push_to_hub_organization:
+    value: null
+push_to_hub_token:
+    value: <PUSH_TO_HUB_TOKEN>
+ray_scope:
+    value: last
+remove_invalid_values:
+    value: false
+remove_unused_columns:
+    value: false
+repetition_penalty:
+    value: 1
+report_to:
+    value:
+        - wandb
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+return_dict_in_generate:
+    value: false
+run_name:
+    value: a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY_steps11000_gpus4_acc1
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_safetensors:
+    value: true
+save_steps:
+    value: 0.5
+save_strategy:
+    value: steps
+save_total_limit:
+    value: 2
+seed:
+    value: 42
+select_layer:
+    value: -1
+sep_token_id:
+    value: null
+skip_memory_metrics:
+    value: true
+split_batches:
+    value: false
+suppress_tokens:
+    value: null
+system_message:
+    value: You are an autonomous navigation agent operating in indoor environments. You receive spatial information through position embeddings injected into visual features and text tokens. Use the BEV map, position embeddings, and semantic information to make navigation decisions. When the target object is detected (<target> marker), navigate directly to it. Otherwise, explore frontiers strategically to find the goal object.
+task_specific_params:
+    value: null
+temperature:
+    value: 1
+template:
+    value: internvl2_5_nav
+tf_legacy_loss:
+    value: false
+tf32:
+    value: null
+tie_encoder_decoder:
+    value: false
+tie_word_embeddings:
+    value: false
+tokenizer_class:
+    value: null
+top_k:
+    value: 50
+top_p:
+    value: 1
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_dtype:
+    value: torch.bfloat16
+torchdynamo:
+    value: null
+torchscript:
+    value: false
+tpu_metrics_debug:
+    value: false
+tpu_num_cores:
+    value: null
+transformers_version:
+    value: null
+typical_p:
+    value: 1
+use_backbone_lora:
+    value: 0
+use_bfloat16:
+    value: false
+use_cpu:
+    value: false
+use_ipex:
+    value: false
+use_legacy_prediction_loop:
+    value: false
+use_llm_lora:
+    value: 64
+use_mps_device:
+    value: false
+use_pairwise_spatial_encoder:
+    value: false
+use_position_embeddings:
+    value: true
+use_thumbnail:
+    value: true
+vision_config:
+    value:
+        _attn_implementation_autoset: true
+        _name_or_path: OpenGVLab/InternViT-6B-448px-V1-5
+        add_cross_attention: false
+        architectures:
+            - InternVisionModel
+        attention_dropout: 0
+        auto_map:
+            AutoConfig: configuration_intern_vit.InternVisionConfig
+            AutoModel: modeling_intern_vit.InternVisionModel
+        bad_words_ids: null
+        begin_suppress_tokens: null
+        bos_token_id: null
+        capacity_factor: 1.2
+        chunk_size_feed_forward: 0
+        cross_attention_hidden_size: null
+        decoder_start_token_id: null
+        diversity_penalty: 0
+        do_sample: false
+        drop_path_rate: 0
+        dropout: 0
+        early_stopping: false
+        encoder_no_repeat_ngram_size: 0
+        eos_token_id: null
+        eval_capacity_factor: 1.4
+        exponential_decay_length_penalty: null
+        finetuning_task: null
+        forced_bos_token_id: null
+        forced_eos_token_id: null
+        hidden_act: gelu
+        hidden_size: 1024
+        id2label:
+            "0": LABEL_0
+            "1": LABEL_1
+        image_size: 448
+        initializer_factor: 0.1
+        initializer_range: 1e-10
+        intermediate_size: 4096
+        is_decoder: false
+        is_encoder_decoder: false
+        label2id:
+            LABEL_0: 0
+            LABEL_1: 1
+        laux_allreduce: all_nodes
+        layer_norm_eps: 1e-06
+        length_penalty: 1
+        max_length: 20
+        min_length: 0
+        model_type: intern_vit_6b
+        moe_coeff_ratio: 0.5
+        moe_intermediate_size: 768
+        moe_output_scale: 4
+        no_repeat_ngram_size: 0
+        noisy_gate_policy: RSample_before
+        norm_type: layer_norm
+        num_attention_heads: 16
+        num_beam_groups: 1
+        num_beams: 1
+        num_channels: 3
+        num_experts: 8
+        num_hidden_layers: 24
+        num_return_sequences: 1
+        num_routed_experts: 4
+        num_shared_experts: 4
+        output_attentions: false
+        output_hidden_states: false
+        output_scores: false
+        pad_token_id: null
+        patch_size: 14
+        prefix: null
+        problem_type: null
+        qk_normalization: false
+        qkv_bias: true
+        remove_invalid_values: false
+        repetition_penalty: 1
+        return_dict: true
+        return_dict_in_generate: false
+        sep_token_id: null
+        shared_expert_intermediate_size: 3072
+        suppress_tokens: null
+        task_specific_params: null
+        temperature: 1
+        tf_legacy_loss: false
+        tie_encoder_decoder: false
+        tie_word_embeddings: true
+        tokenizer_class: null
+        top_k: 50
+        top_p: 1
+        torch_dtype: bfloat16
+        torchscript: false
+        transformers_version: 4.37.2
+        typical_p: 1
+        use_bfloat16: true
+        use_flash_attn: true
+        use_moe: false
+        use_residual: true
+        use_rts: false
+        use_weighted_residual: false
+vit_bev_freeze:
+    value: true
+vit_bev_lora_rank:
+    value: 64
+vit_bev_use_lora:
+    value: true
+vit_rgb_freeze:
+    value: true
+vit_rgb_lora_rank:
+    value: 16
+vit_rgb_use_lora:
+    value: true
+warmup_ratio:
+    value: 0.03
+warmup_steps:
+    value: 0
+weight_decay:
+    value: 0.01

wandb/run-20260225_055938-u2vegsv1/files/output.log

@@ -0,0 +1,1323 @@
+  0%|          | 0/11000 [00:00<?, ?it/s]
+02/25/2026 06:17:33 - INFO - internvl_cleaned.model.internvl_chat.modeling_internvl_chat - [Text Position Embedding Verification] Sample 1:
+  <s> at token 437 sub_idx=0
+  <s> at token 1921 sub_idx=1
+  <s> at token 3160 sub_idx=2
+  <s> at token 4545 sub_idx=3
+  <s> at token 5493 sub_idx=4
+  <s> at token 6587 sub_idx=5
+  <s> at token 7484 sub_idx=6
+  <s> at token 8141 sub_idx=7
+  <s> at token 8945 sub_idx=8
+  <s> at token 9891 sub_idx=9
+  <s> at token 10835 sub_idx=10
+  <s> at token 12123 sub_idx=11
+  <s> at token 13020 sub_idx=12
+  <s> at token 13821 sub_idx=13
+  <s> at token 14575 sub_idx=14
+  <s> at token 15767 sub_idx=15
+  <s> at token 16617 sub_idx=16
+  <s> at token 17905 sub_idx=17
+  <s> at token 19243 sub_idx=18
+  <s> at token 20482 sub_idx=19
+  <s> at token 21331 sub_idx=20
+  <s> at token 22180 sub_idx=21
+  <s> at token 23030 sub_idx=22
+  <s> at token 23973 sub_idx=23
+  <e_s> at token 439 sub_idx=0
+  <e_s> at token 1923 sub_idx=1
+  <e_s> at token 3162 sub_idx=2
+  <e_s> at token 4547 sub_idx=3
+  <e_s> at token 5495 sub_idx=4
+  <e_s> at token 6589 sub_idx=5
+  <e_s> at token 7486 sub_idx=6
+  <e_s> at token 8143 sub_idx=7
+  <e_s> at token 8947 sub_idx=8
+  <e_s> at token 9893 sub_idx=9
+  <e_s> at token 10837 sub_idx=10
+  <e_s> at token 12125 sub_idx=11
+  <e_s> at token 13022 sub_idx=12
+  <e_s> at token 13823 sub_idx=13
+  <e_s> at token 14577 sub_idx=14
+  <e_s> at token 15769 sub_idx=15
+  <e_s> at token 16619 sub_idx=16
+  <e_s> at token 17907 sub_idx=17
+  <e_s> at token 19245 sub_idx=18
+  <e_s> at token 20484 sub_idx=19
+  <e_s> at token 21333 sub_idx=20
+  <e_s> at token 22182 sub_idx=21
+  <e_s> at token 23032 sub_idx=22
+  <e_s> at token 23975 sub_idx=23
+  <cand> 0 at token 444 sub_idx=0 pixel=[320.0,270.0]
+  <cand> 1 at token 492 sub_idx=0 pixel=[185.0,213.0]
+  <cand> 2 at token 540 sub_idx=0 pixel=[319.0,247.0]
+  <cand> 3 at token 588 sub_idx=0 pixel=[256.0,210.0]
+  <cand> 4 at token 636 sub_idx=0 pixel=[212.0,138.0]
+  <cand> 5 at token 684 sub_idx=0 pixel=[245.0,121.0]
+  <cand> 6 at token 732 sub_idx=0 pixel=[272.0,370.0]
+  <cand> 7 at token 780 sub_idx=0 pixel=[307.0,160.0]
+  <cand> 8 at token 828 sub_idx=0 pixel=[242.0,314.0]
+  <cand> 9 at token 876 sub_idx=0 pixel=[181.0,243.0]
+  <cand> 10 at token 924 sub_idx=0 pixel=[235.0,183.0]
+  <cand> 11 at token 973 sub_idx=0 pixel=[275.0,174.0]
+  <cand> 12 at token 1022 sub_idx=0 pixel=[240.0,152.0]
+  <cand> 13 at token 1071 sub_idx=0 pixel=[216.0,214.0]
+  <cand> 14 at token 1118 sub_idx=0 pixel=[315.0,117.0]
+  <cand> 15 at token 1167 sub_idx=0 pixel=[226.0,111.0]
+  <cand> 16 at token 1216 sub_idx=0 pixel=[189.0,277.0]
+  <cand> 17 at token 1265 sub_idx=0 pixel=[278.0,161.0]
+  <cand> 18 at token 1314 sub_idx=0 pixel=[203.0,286.0]
+  <cand> 19 at token 1363 sub_idx=0 pixel=[275.0,194.0]
+  <cand> 20 at token 1412 sub_idx=0 pixel=[404.0,400.0]
+  <cand> 0 at token 1928 sub_idx=1 pixel=[141.0,126.0]
+  <cand> 1 at token 1976 sub_idx=1 pixel=[109.0,55.0]
+  <cand> 2 at token 2024 sub_idx=1 pixel=[143.0,246.0]
+  <cand> 3 at token 2072 sub_idx=1 pixel=[224.0,269.0]
+  <cand> 4 at token 2120 sub_idx=1 pixel=[145.0,219.0]
+  <cand> 5 at token 2168 sub_idx=1 pixel=[175.0,188.0]
+  <cand> 6 at token 2216 sub_idx=1 pixel=[45.0,88.0]
+  <cand> 7 at token 2264 sub_idx=1 pixel=[57.0,178.0]
+  <cand> 8 at token 2312 sub_idx=1 pixel=[18.0,16.0]
+  <cand> 9 at token 2360 sub_idx=1 pixel=[97.0,76.0]
+  <cand> 10 at token 2408 sub_idx=1 pixel=[24.0,147.0]
+  <cand> 11 at token 2457 sub_idx=1 pixel=[161.0,107.0]
+  <cand> 12 at token 2506 sub_idx=1 pixel=[233.0,297.0]
+  <cand> 13 at token 2553 sub_idx=1 pixel=[100.0,222.0]
+  <cand> 14 at token 2602 sub_idx=1 pixel=[31.0,226.0]
+  <cand> 15 at token 2651 sub_idx=1 pixel=[224.0,255.0]
+  <cand> 0 at token 3167 sub_idx=2 pixel=[155.0,57.0]
+  <cand> 1 at token 3215 sub_idx=2 pixel=[6.0,64.0]
+  <cand> 2 at token 3263 sub_idx=2 pixel=[172.0,64.0]
+  <cand> 3 at token 3311 sub_idx=2 pixel=[174.0,148.0]
+  <cand> 4 at token 3359 sub_idx=2 pixel=[20.0,108.0]
+  <cand> 5 at token 3407 sub_idx=2 pixel=[86.0,136.0]
+  <cand> 6 at token 3455 sub_idx=2 pixel=[84.0,33.0]
+  <cand> 7 at token 3503 sub_idx=2 pixel=[91.0,69.0]
+  <cand> 8 at token 3551 sub_idx=2 pixel=[219.0,232.0]
+  <cand> 9 at token 3597 sub_idx=2 pixel=[28.0,85.0]
+  <cand> 10 at token 3645 sub_idx=2 pixel=[195.0,236.0]
+  <cand> 11 at token 3694 sub_idx=2 pixel=[29.0,44.0]
+  <cand> 12 at token 3743 sub_idx=2 pixel=[157.0,76.0]
+  <cand> 13 at token 3792 sub_idx=2 pixel=[38.0,129.0]
+  <cand> 14 at token 3841 sub_idx=2 pixel=[86.0,208.0]
+  <cand> 15 at token 3890 sub_idx=2 pixel=[168.0,211.0]
+  <cand> 16 at token 3939 sub_idx=2 pixel=[49.0,206.0]
+  <cand> 17 at token 3988 sub_idx=2 pixel=[149.0,230.0]
+  <cand> 18 at token 4037 sub_idx=2 pixel=[58.0,102.0]
+  <cand> 0 at token 4552 sub_idx=3 pixel=[21.0,189.0]
+  <cand> 1 at token 4600 sub_idx=3 pixel=[220.0,170.0]
+  <cand> 2 at token 4648 sub_idx=3 pixel=[5.0,225.0]
+  <cand> 3 at token 4696 sub_idx=3 pixel=[193.0,118.0]
+  <cand> 4 at token 4744 sub_idx=3 pixel=[180.0,151.0]
+  <cand> 5 at token 4792 sub_idx=3 pixel=[72.0,83.0]
+  <cand> 6 at token 4840 sub_idx=3 pixel=[157.0,221.0]
+  <cand> 7 at token 4888 sub_idx=3 pixel=[159.0,82.0]
+  <cand> 8 at token 4936 sub_idx=3 pixel=[99.0,179.0]
+  <cand> 9 at token 4984 sub_idx=3 pixel=[213.0,234.0]
+  <cand> 0 at token 5500 sub_idx=4 pixel=[278.0,132.0]
+  <cand> 1 at token 5548 sub_idx=4 pixel=[272.0,71.0]
+  <cand> 2 at token 5596 sub_idx=4 pixel=[278.0,266.0]
+  <cand> 3 at token 5644 sub_idx=4 pixel=[222.0,135.0]
+  <cand> 4 at token 5692 sub_idx=4 pixel=[208.0,113.0]
+  <cand> 5 at token 5740 sub_idx=4 pixel=[260.0,237.0]
+  <cand> 6 at token 5788 sub_idx=4 pixel=[175.0,258.0]
+  <cand> 7 at token 5836 sub_idx=4 pixel=[229.0,282.0]
+  <cand> 8 at token 5884 sub_idx=4 pixel=[291.0,200.0]
+  <cand> 9 at token 5932 sub_idx=4 pixel=[235.0,65.0]
+  <cand> 10 at token 5980 sub_idx=4 pixel=[269.0,287.0]
+  <cand> 11 at token 6029 sub_idx=4 pixel=[207.0,86.0]
+  <cand> 12 at token 6078 sub_idx=4 pixel=[183.0,242.0]
+  <cand> 0 at token 6594 sub_idx=5 pixel=[312.0,203.0]
+  <cand> 1 at token 6642 sub_idx=5 pixel=[203.0,237.0]
+  <cand> 2 at token 6690 sub_idx=5 pixel=[230.0,143.0]
+  <cand> 3 at token 6738 sub_idx=5 pixel=[342.0,209.0]
+  <cand> 4 at token 6786 sub_idx=5 pixel=[247.0,178.0]
+  <cand> 5 at token 6834 sub_idx=5 pixel=[216.0,115.0]
+  <cand> 6 at token 6882 sub_idx=5 pixel=[308.0,249.0]
+  <cand> 7 at token 6930 sub_idx=5 pixel=[263.0,258.0]
+  <cand> 8 at token 6978 sub_idx=5 pixel=[218.0,197.0]
+  <cand> 0 at token 7491 sub_idx=6 pixel=[173.0,163.0]
+  <cand> 1 at token 7539 sub_idx=6 pixel=[253.0,129.0]
+  <cand> 2 at token 7587 sub_idx=6 pixel=[192.0,203.0]
+  <cand> 3 at token 7635 sub_idx=6 pixel=[193.0,115.0]
+  <cand> 0 at token 8148 sub_idx=7 pixel=[269.0,204.0]
+  <cand> 1 at token 8196 sub_idx=7 pixel=[161.0,221.0]
+  <cand> 2 at token 8244 sub_idx=7 pixel=[193.0,296.0]
+  <cand> 3 at token 8292 sub_idx=7 pixel=[159.0,251.0]
+  <cand> 4 at token 8340 sub_idx=7 pixel=[139.0,222.0]
+  <cand> 5 at token 8388 sub_idx=7 pixel=[271.0,231.0]
+  <cand> 6 at token 8436 sub_idx=7 pixel=[242.0,235.0]
+  <cand> 0 at token 8952 sub_idx=8 pixel=[234.0,193.0]
+  <cand> 1 at token 9000 sub_idx=8 pixel=[211.0,399.0]
+  <cand> 2 at token 9048 sub_idx=8 pixel=[204.0,379.0]
+  <cand> 3 at token 9096 sub_idx=8 pixel=[217.0,126.0]
+  <cand> 4 at token 9144 sub_idx=8 pixel=[197.0,206.0]
+  <cand> 5 at token 9192 sub_idx=8 pixel=[272.0,344.0]
+  <cand> 6 at token 9240 sub_idx=8 pixel=[241.0,177.0]
+  <cand> 7 at token 9288 sub_idx=8 pixel=[303.0,416.0]
+  <cand> 8 at token 9336 sub_idx=8 pixel=[194.0,316.0]
+  <cand> 9 at token 9384 sub_idx=8 pixel=[269.0,443.0]
+  <cand> 0 at token 9898 sub_idx=9 pixel=[248.0,271.0]
+  <cand> 1 at token 9946 sub_idx=9 pixel=[183.0,251.0]
+  <cand> 2 at token 9994 sub_idx=9 pixel=[243.0,244.0]
+  <cand> 3 at token 10042 sub_idx=9 pixel=[251.0,312.0]
+  <cand> 4 at token 10090 sub_idx=9 pixel=[230.0,214.0]
+  <cand> 5 at token 10136 sub_idx=9 pixel=[107.0,373.0]
+  <cand> 6 at token 10184 sub_idx=9 pixel=[110.0,350.0]
+  <cand> 7 at token 10232 sub_idx=9 pixel=[324.0,326.0]
+  <cand> 8 at token 10280 sub_idx=9 pixel=[286.0,375.0]
+  <cand> 9 at token 10328 sub_idx=9 pixel=[331.0,346.0]
+  <cand> 0 at token 10842 sub_idx=10 pixel=[317.0,192.0]
+  <cand> 1 at token 10890 sub_idx=10 pixel=[110.0,244.0]
+  <cand> 2 at token 10938 sub_idx=10 pixel=[246.0,181.0]
+  <cand> 3 at token 10986 sub_idx=10 pixel=[209.0,249.0]
+  <cand> 4 at token 11034 sub_idx=10 pixel=[230.0,268.0]
+  <cand> 5 at token 11082 sub_idx=10 pixel=[144.0,222.0]
+  <cand> 6 at token 11130 sub_idx=10 pixel=[276.0,212.0]
+  <cand> 7 at token 11178 sub_idx=10 pixel=[138.0,293.0]
+  <cand> 8 at token 11226 sub_idx=10 pixel=[189.0,298.0]
+  <cand> 9 at token 11274 sub_idx=10 pixel=[150.0,237.0]
+  <cand> 10 at token 11322 sub_idx=10 pixel=[88.0,340.0]
+  <cand> 11 at token 11371 sub_idx=10 pixel=[49.0,330.0]
+  <cand> 12 at token 11420 sub_idx=10 pixel=[128.0,273.0]
+  <cand> 13 at token 11469 sub_idx=10 pixel=[244.0,222.0]
+  <cand> 14 at token 11516 sub_idx=10 pixel=[22.0,204.0]
+  <cand> 15 at token 11565 sub_idx=10 pixel=[223.0,311.0]
+  <cand> 16 at token 11614 sub_idx=10 pixel=[295.0,180.0]
+  <cand> 0 at token 12130 sub_idx=11 pixel=[255.0,239.0]
+  <cand> 1 at token 12178 sub_idx=11 pixel=[129.0,321.0]
+  <cand> 2 at token 12226 sub_idx=11 pixel=[299.0,209.0]
+  <cand> 3 at token 12274 sub_idx=11 pixel=[255.0,210.0]
+  <cand> 4 at token 12322 sub_idx=11 pixel=[109.0,327.0]
+  <cand> 5 at token 12370 sub_idx=11 pixel=[239.0,296.0]
+  <cand> 6 at token 12418 sub_idx=11 pixel=[136.0,291.0]
+  <cand> 7 at token 12466 sub_idx=11 pixel=[211.0,319.0]
+  <cand> 8 at token 12514 sub_idx=11 pixel=[171.0,313.0]
+  <cand> 0 at token 13027 sub_idx=12 pixel=[261.0,348.0]
+  <cand> 1 at token 13075 sub_idx=12 pixel=[321.0,357.0]
+  <cand> 2 at token 13123 sub_idx=12 pixel=[171.0,274.0]
+  <cand> 3 at token 13171 sub_idx=12 pixel=[225.0,182.0]
+  <cand> 4 at token 13219 sub_idx=12 pixel=[270.0,115.0]
+  <cand> 5 at token 13267 sub_idx=12 pixel=[328.0,265.0]
+  <cand> 6 at token 13315 sub_idx=12 pixel=[224.0,161.0]
+  <cand> 0 at token 13828 sub_idx=13 pixel=[221.0,187.0]
+  <cand> 1 at token 13876 sub_idx=13 pixel=[189.0,204.0]
+  <cand> 2 at token 13924 sub_idx=13 pixel=[220.0,160.0]
+  <cand> 3 at token 13972 sub_idx=13 pixel=[194.0,236.0]
+  <cand> 4 at token 14020 sub_idx=13 pixel=[150.0,247.0]
+  <cand> 5 at token 14068 sub_idx=13 pixel=[210.0,197.0]
+  <cand> 0 at token 14582 sub_idx=14 pixel=[219.0,191.0]
+  <cand> 1 at token 14630 sub_idx=14 pixel=[150.0,237.0]
+  <cand> 2 at token 14678 sub_idx=14 pixel=[266.0,213.0]
+  <cand> 3 at token 14726 sub_idx=14 pixel=[196.0,129.0]
+  <cand> 4 at token 14774 sub_idx=14 pixel=[213.0,347.0]
+  <cand> 5 at token 14822 sub_idx=14 pixel=[173.0,265.0]
+  <cand> 6 at token 14870 sub_idx=14 pixel=[215.0,274.0]
+  <cand> 7 at token 14918 sub_idx=14 pixel=[165.0,210.0]
+  <cand> 8 at token 14966 sub_idx=14 pixel=[264.0,279.0]
+  <cand> 9 at token 15014 sub_idx=14 pixel=[155.0,191.0]
+  <cand> 10 at token 15062 sub_idx=14 pixel=[373.0,380.0]
+  <cand> 11 at token 15111 sub_idx=14 pixel=[136.0,265.0]
+  <cand> 12 at token 15160 sub_idx=14 pixel=[223.0,259.0]
+  <cand> 13 at token 15209 sub_idx=14 pixel=[292.0,249.0]
+  <cand> 14 at token 15258 sub_idx=14 pixel=[357.0,387.0]
+  <cand> 0 at token 15774 sub_idx=15 pixel=[249.0,86.0]
+  <cand> 1 at token 15822 sub_idx=15 pixel=[173.0,218.0]
+  <cand> 2 at token 15870 sub_idx=15 pixel=[280.0,249.0]
+  <cand> 3 at token 15918 sub_idx=15 pixel=[288.0,211.0]
+  <cand> 4 at token 15966 sub_idx=15 pixel=[228.0,82.0]
+  <cand> 5 at token 16014 sub_idx=15 pixel=[310.0,161.0]
+  <cand> 6 at token 16062 sub_idx=15 pixel=[178.0,234.0]
+  <cand> 7 at token 16110 sub_idx=15 pixel=[232.0,51.0]
+  <cand> 0 at token 16624 sub_idx=16 pixel=[104.0,112.0]
+  <cand> 1 at token 16672 sub_idx=16 pixel=[182.0,47.0]
+  <cand> 2 at token 16720 sub_idx=16 pixel=[189.0,245.0]
+  <cand> 3 at token 16768 sub_idx=16 pixel=[245.0,57.0]
+  <cand> 4 at token 16816 sub_idx=16 pixel=[325.0,61.0]
+  <cand> 5 at token 16864 sub_idx=16 pixel=[108.0,230.0]
+  <cand> 6 at token 16912 sub_idx=16 pixel=[98.0,101.0]
+  <cand> 7 at token 16960 sub_idx=16 pixel=[365.0,224.0]
+  <cand> 8 at token 17008 sub_idx=16 pixel=[223.0,175.0]
+  <cand> 9 at token 17056 sub_idx=16 pixel=[260.0,311.0]
+  <cand> 10 at token 17104 sub_idx=16 pixel=[216.0,251.0]
+  <cand> 11 at token 17151 sub_idx=16 pixel=[284.0,246.0]
+  <cand> 12 at token 17200 sub_idx=16 pixel=[206.0,195.0]
+  <cand> 13 at token 17249 sub_idx=16 pixel=[339.0,258.0]
+  <cand> 14 at token 17298 sub_idx=16 pixel=[302.0,249.0]
+  <cand> 15 at token 17347 sub_idx=16 pixel=[316.0,177.0]
+  <cand> 16 at token 17396 sub_idx=16 pixel=[271.0,154.0]
+  <cand> 0 at token 17912 sub_idx=17 pixel=[281.0,280.0]
+  <cand> 1 at token 17960 sub_idx=17 pixel=[250.0,245.0]
+  <cand> 2 at token 18008 sub_idx=17 pixel=[197.0,321.0]
+  <cand> 3 at token 18056 sub_idx=17 pixel=[188.0,271.0]
+  <cand> 4 at token 18104 sub_idx=17 pixel=[383.0,250.0]
+  <cand> 5 at token 18152 sub_idx=17 pixel=[363.0,189.0]
+  <cand> 6 at token 18200 sub_idx=17 pixel=[206.0,180.0]
+  <cand> 7 at token 18248 sub_idx=17 pixel=[305.0,327.0]
+  <cand> 8 at token 18296 sub_idx=17 pixel=[244.0,157.0]
+  <cand> 9 at token 18344 sub_idx=17 pixel=[206.0,128.0]
+  <cand> 10 at token 18392 sub_idx=17 pixel=[365.0,172.0]
+  <cand> 11 at token 18441 sub_idx=17 pixel=[174.0,295.0]
+  <cand> 12 at token 18490 sub_idx=17 pixel=[355.0,309.0]
+  <cand> 13 at token 18539 sub_idx=17 pixel=[240.0,235.0]
+  <cand> 14 at token 18586 sub_idx=17 pixel=[153.0,326.0]
+  <cand> 15 at token 18635 sub_idx=17 pixel=[238.0,325.0]
+  <cand> 16 at token 18684 sub_idx=17 pixel=[364.0,327.0]
+  <cand> 17 at token 18733 sub_idx=17 pixel=[179.0,248.0]
+  <cand> 0 at token 19250 sub_idx=18 pixel=[196.0,286.0]
+  <cand> 1 at token 19298 sub_idx=18 pixel=[186.0,308.0]
+  <cand> 2 at token 19346 sub_idx=18 pixel=[354.0,318.0]
+  <cand> 3 at token 19394 sub_idx=18 pixel=[196.0,186.0]
+  <cand> 4 at token 19442 sub_idx=18 pixel=[212.0,193.0]
+  <cand> 5 at token 19490 sub_idx=18 pixel=[205.0,259.0]
+  <cand> 6 at token 19538 sub_idx=18 pixel=[321.0,249.0]
+  <cand> 7 at token 19586 sub_idx=18 pixel=[236.0,238.0]
+  <cand> 8 at token 19634 sub_idx=18 pixel=[323.0,267.0]
+  <cand> 9 at token 19682 sub_idx=18 pixel=[152.0,188.0]
+  <cand> 10 at token 19730 sub_idx=18 pixel=[305.0,190.0]
+  <cand> 11 at token 19779 sub_idx=18 pixel=[237.0,268.0]
+  <cand> 12 at token 19826 sub_idx=18 pixel=[143.0,203.0]
+  <cand> 13 at token 19875 sub_idx=18 pixel=[226.0,280.0]
+  <cand> 14 at token 19924 sub_idx=18 pixel=[237.0,320.0]
+  <cand> 15 at token 19973 sub_idx=18 pixel=[188.0,183.0]
+  <cand> 0 at token 20489 sub_idx=19 pixel=[197.0,269.0]
+  <cand> 1 at token 20537 sub_idx=19 pixel=[227.0,291.0]
+  <cand> 2 at token 20585 sub_idx=19 pixel=[209.0,204.0]
+  <cand> 3 at token 20633 sub_idx=19 pixel=[230.0,264.0]
+  <cand> 4 at token 20681 sub_idx=19 pixel=[247.0,174.0]
+  <cand> 5 at token 20729 sub_idx=19 pixel=[246.0,194.0]
+  <cand> 6 at token 20777 sub_idx=19 pixel=[264.0,226.0]
+  <cand> 7 at token 20825 sub_idx=19 pixel=[260.0,217.0]
+  <cand> 0 at token 21338 sub_idx=20 pixel=[237.0,243.0]
+  <cand> 1 at token 21384 sub_idx=20 pixel=[277.0,77.0]
+  <cand> 2 at token 21432 sub_idx=20 pixel=[310.0,237.0]
+  <cand> 3 at token 21480 sub_idx=20 pixel=[328.0,289.0]
+  <cand> 4 at token 21528 sub_idx=20 pixel=[245.0,101.0]
+  <cand> 5 at token 21576 sub_idx=20 pixel=[241.0,159.0]
+  <cand> 6 at token 21624 sub_idx=20 pixel=[183.0,200.0]
+  <cand> 7 at token 21672 sub_idx=20 pixel=[229.0,270.0]
+  <cand> 0 at token 22187 sub_idx=21 pixel=[134.0,124.0]
+  <cand> 1 at token 22235 sub_idx=21 pixel=[195.0,137.0]
+  <cand> 2 at token 22283 sub_idx=21 pixel=[65.0,112.0]
+  <cand> 3 at token 22331 sub_idx=21 pixel=[255.0,264.0]
+  <cand> 4 at token 22379 sub_idx=21 pixel=[207.0,281.0]
+  <cand> 5 at token 22427 sub_idx=21 pixel=[97.0,122.0]
+  <cand> 6 at token 22475 sub_idx=21 pixel=[241.0,182.0]
+  <cand> 7 at token 22523 sub_idx=21 pixel=[226.0,183.0]
+  <cand> 0 at token 23037 sub_idx=22 pixel=[242.0,288.0]
+  <cand> 1 at token 23085 sub_idx=22 pixel=[113.0,419.0]
+  <cand> 2 at token 23133 sub_idx=22 pixel=[218.0,220.0]
+  <cand> 3 at token 23179 sub_idx=22 pixel=[142.0,145.0]
+  <cand> 4 at token 23227 sub_idx=22 pixel=[218.0,368.0]
+  <cand> 5 at token 23275 sub_idx=22 pixel=[208.0,266.0]
+  <cand> 6 at token 23323 sub_idx=22 pixel=[198.0,377.0]
+  <cand> 7 at token 23371 sub_idx=22 pixel=[122.0,372.0]
+  <cand> 8 at token 23419 sub_idx=22 pixel=[226.0,278.0]
+  <cand> 9 at token 23467 sub_idx=22 pixel=[239.0,278.0]
+  <cand> 0 at token 23980 sub_idx=23 pixel=[197.0,39.0]
+  <cand> 1 at token 24028 sub_idx=23 pixel=[290.0,274.0]
+  <cand> 2 at token 24076 sub_idx=23 pixel=[228.0,337.0]
+  <cand> 3 at token 24124 sub_idx=23 pixel=[259.0,296.0]
+  <cand> 4 at token 24172 sub_idx=23 pixel=[267.0,247.0]
+  <cand> 5 at token 24220 sub_idx=23 pixel=[185.0,32.0]
+  <cand> 6 at token 24268 sub_idx=23 pixel=[318.0,213.0]
+  <cand> 7 at token 24316 sub_idx=23 pixel=[316.0,38.0]
+  <cand> 8 at token 24364 sub_idx=23 pixel=[267.0,316.0]
+  <cand> 9 at token 24412 sub_idx=23 pixel=[209.0,218.0]
+  <e_cand> 0 at token 490 sub_idx=0 (same embed)
+  <e_cand> 1 at token 538 sub_idx=0 (same embed)
+  <e_cand> 2 at token 586 sub_idx=0 (same embed)
+  <e_cand> 0 at token 1974 sub_idx=1 (same embed)
+  <e_cand> 1 at token 2022 sub_idx=1 (same embed)
+  <e_cand> 2 at token 2070 sub_idx=1 (same embed)
+  <e_cand> 0 at token 3213 sub_idx=2 (same embed)
+  <e_cand> 1 at token 3261 sub_idx=2 (same embed)
+  <e_cand> 2 at token 3309 sub_idx=2 (same embed)
+  <e_cand> 0 at token 4598 sub_idx=3 (same embed)
+  <e_cand> 1 at token 4646 sub_idx=3 (same embed)
+  <e_cand> 2 at token 4694 sub_idx=3 (same embed)
+  <e_cand> 0 at token 5546 sub_idx=4 (same embed)
+  <e_cand> 1 at token 5594 sub_idx=4 (same embed)
+  <e_cand> 2 at token 5642 sub_idx=4 (same embed)
+  <e_cand> 0 at token 6640 sub_idx=5 (same embed)
+  <e_cand> 1 at token 6688 sub_idx=5 (same embed)
+  <e_cand> 2 at token 6736 sub_idx=5 (same embed)
+  <e_cand> 0 at token 7537 sub_idx=6 (same embed)
+  <e_cand> 1 at token 7585 sub_idx=6 (same embed)
+  <e_cand> 2 at token 7633 sub_idx=6 (same embed)
+  <e_cand> 0 at token 8194 sub_idx=7 (same embed)
+  <e_cand> 1 at token 8242 sub_idx=7 (same embed)
+  <e_cand> 2 at token 8290 sub_idx=7 (same embed)
+  <e_cand> 0 at token 8998 sub_idx=8 (same embed)
+  <e_cand> 1 at token 9046 sub_idx=8 (same embed)
+  <e_cand> 2 at token 9094 sub_idx=8 (same embed)
+  <e_cand> 0 at token 9944 sub_idx=9 (same embed)
+  <e_cand> 1 at token 9992 sub_idx=9 (same embed)
+  <e_cand> 2 at token 10040 sub_idx=9 (same embed)
+  <e_cand> 0 at token 10888 sub_idx=10 (same embed)
+  <e_cand> 1 at token 10936 sub_idx=10 (same embed)
+  <e_cand> 2 at token 10984 sub_idx=10 (same embed)
+  <e_cand> 0 at token 12176 sub_idx=11 (same embed)
+  <e_cand> 1 at token 12224 sub_idx=11 (same embed)
+  <e_cand> 2 at token 12272 sub_idx=11 (same embed)
+  <e_cand> 0 at token 13073 sub_idx=12 (same embed)
+  <e_cand> 1 at token 13121 sub_idx=12 (same embed)
+  <e_cand> 2 at token 13169 sub_idx=12 (same embed)
+  <e_cand> 0 at token 13874 sub_idx=13 (same embed)
+  <e_cand> 1 at token 13922 sub_idx=13 (same embed)
+  <e_cand> 2 at token 13970 sub_idx=13 (same embed)
+  <e_cand> 0 at token 14628 sub_idx=14 (same embed)
+  <e_cand> 1 at token 14676 sub_idx=14 (same embed)
+  <e_cand> 2 at token 14724 sub_idx=14 (same embed)
+  <e_cand> 0 at token 15820 sub_idx=15 (same embed)
+  <e_cand> 1 at token 15868 sub_idx=15 (same embed)
+  <e_cand> 2 at token 15916 sub_idx=15 (same embed)
+  <e_cand> 0 at token 16670 sub_idx=16 (same embed)
+  <e_cand> 1 at token 16718 sub_idx=16 (same embed)
+  <e_cand> 2 at token 16766 sub_idx=16 (same embed)
+  <e_cand> 0 at token 17958 sub_idx=17 (same embed)
+  <e_cand> 1 at token 18006 sub_idx=17 (same embed)
+  <e_cand> 2 at token 18054 sub_idx=17 (same embed)
+  <e_cand> 0 at token 19296 sub_idx=18 (same embed)
+  <e_cand> 1 at token 19344 sub_idx=18 (same embed)
+  <e_cand> 2 at token 19392 sub_idx=18 (same embed)
+  <e_cand> 0 at token 20535 sub_idx=19 (same embed)
+  <e_cand> 1 at token 20583 sub_idx=19 (same embed)
+  <e_cand> 2 at token 20631 sub_idx=19 (same embed)
+  <e_cand> 0 at token 21382 sub_idx=20 (same embed)
+  <e_cand> 1 at token 21430 sub_idx=20 (same embed)
+  <e_cand> 2 at token 21478 sub_idx=20 (same embed)
+  <e_cand> 0 at token 22233 sub_idx=21 (same embed)
+  <e_cand> 1 at token 22281 sub_idx=21 (same embed)
+  <e_cand> 2 at token 22329 sub_idx=21 (same embed)
+  <e_cand> 0 at token 23083 sub_idx=22 (same embed)
+  <e_cand> 1 at token 23131 sub_idx=22 (same embed)
+  <e_cand> 2 at token 23177 sub_idx=22 (same embed)
+  <e_cand> 0 at token 24026 sub_idx=23 (same embed)
+  <e_cand> 1 at token 24074 sub_idx=23 (same embed)
+  <e_cand> 2 at token 24122 sub_idx=23 (same embed)
+
+[2026-02-25 06:17:41,876] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 4.88 | optimizer_gradients: 32.07 | optimizer_step: 4.90
+[2026-02-25 06:17:41,878] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 13975.93 | bwd_microstep: 7137.28 | bwd_inner_microstep: 6532.16 | bwd_allreduce_microstep: 605.00 | step_microstep: 143.00
+[2026-02-25 06:17:41,879] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 13975.93 | bwd: 7137.29 | bwd_inner: 6532.15 | bwd_allreduce: 605.03 | step: 143.16
+{'loss': 1.6229, 'learning_rate': 3.0303030303030305e-07, 'epoch': 0.0}
+02/25/2026 06:17:49 - INFO - internvl_cleaned.model.internvl_chat.modeling_internvl_chat - [Text Position Embedding Verification] Sample 2:
+  <s> at token 437 sub_idx=0
+  <s> at token 1336 sub_idx=1
+  <s> at token 2476 sub_idx=2
+  <s> at token 3469 sub_idx=3
+  <s> at token 4463 sub_idx=4
+  <s> at token 5656 sub_idx=5
+  <s> at token 6506 sub_idx=6
+  <s> at token 7695 sub_idx=7
+  <s> at token 8593 sub_idx=8
+  <s> at token 9635 sub_idx=9
+  <s> at token 10245 sub_idx=10
+  <s> at token 10950 sub_idx=11
+  <s> at token 12334 sub_idx=12
+  <s> at token 13376 sub_idx=13
+  <s> at token 14468 sub_idx=14
+  <s> at token 15414 sub_idx=15
+  <s> at token 16023 sub_idx=16
+  <s> at token 17508 sub_idx=17
+  <s> at token 18168 sub_idx=18
+  <s> at token 19309 sub_idx=19
+  <s> at token 20254 sub_idx=20
+  <s> at token 21055 sub_idx=21
+  <s> at token 22001 sub_idx=22
+  <s> at token 23336 sub_idx=23
+  <s> at token 23993 sub_idx=24
+  <e_s> at token 439 sub_idx=0
+  <e_s> at token 1338 sub_idx=1
+  <e_s> at token 2478 sub_idx=2
+  <e_s> at token 3471 sub_idx=3
+  <e_s> at token 4465 sub_idx=4
+  <e_s> at token 5658 sub_idx=5
+  <e_s> at token 6508 sub_idx=6
+  <e_s> at token 7697 sub_idx=7
+  <e_s> at token 8595 sub_idx=8
+  <e_s> at token 9637 sub_idx=9
+  <e_s> at token 10247 sub_idx=10
+  <e_s> at token 10952 sub_idx=11
+  <e_s> at token 12336 sub_idx=12
+  <e_s> at token 13378 sub_idx=13
+  <e_s> at token 14470 sub_idx=14
+  <e_s> at token 15416 sub_idx=15
+  <e_s> at token 16025 sub_idx=16
+  <e_s> at token 17510 sub_idx=17
+  <e_s> at token 18170 sub_idx=18
+  <e_s> at token 19311 sub_idx=19
+  <e_s> at token 20256 sub_idx=20
+  <e_s> at token 21057 sub_idx=21
+  <e_s> at token 22003 sub_idx=22
+  <e_s> at token 23338 sub_idx=23
+  <e_s> at token 23995 sub_idx=24
+  <cand> 0 at token 444 sub_idx=0 pixel=[234.0,176.0]
+  <cand> 1 at token 492 sub_idx=0 pixel=[340.0,233.0]
+  <cand> 2 at token 540 sub_idx=0 pixel=[352.0,297.0]
+  <cand> 3 at token 588 sub_idx=0 pixel=[369.0,275.0]
+  <cand> 4 at token 636 sub_idx=0 pixel=[174.0,246.0]
+  <cand> 5 at token 684 sub_idx=0 pixel=[191.0,156.0]
+  <cand> 6 at token 732 sub_idx=0 pixel=[235.0,127.0]
+  <cand> 7 at token 780 sub_idx=0 pixel=[120.0,228.0]
+  <cand> 8 at token 828 sub_idx=0 pixel=[153.0,252.0]
+  <cand> 0 at token 1343 sub_idx=1 pixel=[175.0,236.0]
+  <cand> 1 at token 1389 sub_idx=1 pixel=[140.0,179.0]
+  <cand> 2 at token 1437 sub_idx=1 pixel=[199.0,212.0]
+  <cand> 3 at token 1485 sub_idx=1 pixel=[92.0,241.0]
+  <cand> 4 at token 1533 sub_idx=1 pixel=[207.0,289.0]
+  <cand> 5 at token 1581 sub_idx=1 pixel=[51.0,44.0]
+  <cand> 6 at token 1629 sub_idx=1 pixel=[137.0,286.0]
+  <cand> 7 at token 1677 sub_idx=1 pixel=[155.0,302.0]
+  <cand> 8 at token 1725 sub_idx=1 pixel=[50.0,123.0]
+  <cand> 9 at token 1773 sub_idx=1 pixel=[184.0,187.0]
+  <cand> 10 at token 1821 sub_idx=1 pixel=[159.0,260.0]
+  <cand> 11 at token 1870 sub_idx=1 pixel=[85.0,105.0]
+  <cand> 12 at token 1919 sub_idx=1 pixel=[131.0,122.0]
+  <cand> 13 at token 1968 sub_idx=1 pixel=[111.0,246.0]
+  <cand> 0 at token 2483 sub_idx=2 pixel=[259.0,253.0]
+  <cand> 1 at token 2529 sub_idx=2 pixel=[73.0,162.0]
+  <cand> 2 at token 2577 sub_idx=2 pixel=[287.0,247.0]
+  <cand> 3 at token 2625 sub_idx=2 pixel=[7.0,159.0]
+  <cand> 4 at token 2673 sub_idx=2 pixel=[223.0,267.0]
+  <cand> 5 at token 2721 sub_idx=2 pixel=[274.0,257.0]
+  <cand> 6 at token 2769 sub_idx=2 pixel=[252.0,223.0]
+  <cand> 7 at token 2817 sub_idx=2 pixel=[116.0,118.0]
+  <cand> 8 at token 2865 sub_idx=2 pixel=[13.0,136.0]
+  <cand> 9 at token 2913 sub_idx=2 pixel=[191.0,153.0]
+  <cand> 10 at token 2961 sub_idx=2 pixel=[119.0,147.0]
+  <cand> 0 at token 3476 sub_idx=3 pixel=[170.0,237.0]
+  <cand> 1 at token 3524 sub_idx=3 pixel=[128.0,180.0]
+  <cand> 2 at token 3572 sub_idx=3 pixel=[319.0,253.0]
+  <cand> 3 at token 3620 sub_idx=3 pixel=[305.0,267.0]
+  <cand> 4 at token 3668 sub_idx=3 pixel=[235.0,238.0]
+  <cand> 5 at token 3716 sub_idx=3 pixel=[160.0,162.0]
+  <cand> 6 at token 3764 sub_idx=3 pixel=[122.0,222.0]
+  <cand> 7 at token 3812 sub_idx=3 pixel=[210.0,199.0]
+  <cand> 8 at token 3860 sub_idx=3 pixel=[128.0,234.0]
+  <cand> 9 at token 3908 sub_idx=3 pixel=[311.0,165.0]
+  <cand> 10 at token 3956 sub_idx=3 pixel=[298.0,105.0]
+  <cand> 0 at token 4470 sub_idx=4 pixel=[240.0,286.0]
+  <cand> 1 at token 4518 sub_idx=4 pixel=[79.0,332.0]
+  <cand> 2 at token 4566 sub_idx=4 pixel=[253.0,233.0]
+  <cand> 3 at token 4614 sub_idx=4 pixel=[304.0,152.0]
+  <cand> 4 at token 4662 sub_idx=4 pixel=[253.0,181.0]
+  <cand> 5 at token 4710 sub_idx=4 pixel=[125.0,204.0]
+  <cand> 6 at token 4758 sub_idx=4 pixel=[201.0,319.0]
+  <cand> 7 at token 4806 sub_idx=4 pixel=[136.0,182.0]
+  <cand> 8 at token 4854 sub_idx=4 pixel=[50.0,331.0]
+  <cand> 9 at token 4902 sub_idx=4 pixel=[252.0,159.0]
+  <cand> 10 at token 4950 sub_idx=4 pixel=[126.0,240.0]
+  <cand> 11 at token 4999 sub_idx=4 pixel=[302.0,279.0]
+  <cand> 12 at token 5048 sub_idx=4 pixel=[165.0,304.0]
+  <cand> 13 at token 5097 sub_idx=4 pixel=[225.0,173.0]
+  <cand> 14 at token 5146 sub_idx=4 pixel=[282.0,164.0]
+  <cand> 0 at token 5663 sub_idx=5 pixel=[335.0,446.0]
+  <cand> 1 at token 5711 sub_idx=5 pixel=[200.0,223.0]
+  <cand> 2 at token 5759 sub_idx=5 pixel=[200.0,343.0]
+  <cand> 3 at token 5807 sub_idx=5 pixel=[251.0,326.0]
+  <cand> 4 at token 5855 sub_idx=5 pixel=[293.0,424.0]
+  <cand> 5 at token 5903 sub_idx=5 pixel=[250.0,217.0]
+  <cand> 6 at token 5951 sub_idx=5 pixel=[270.0,198.0]
+  <cand> 7 at token 5999 sub_idx=5 pixel=[310.0,410.0]
+  <cand> 0 at token 6513 sub_idx=6 pixel=[337.0,257.0]
+  <cand> 1 at token 6561 sub_idx=6 pixel=[160.0,200.0]
+  <cand> 2 at token 6609 sub_idx=6 pixel=[219.0,280.0]
+  <cand> 3 at token 6657 sub_idx=6 pixel=[155.0,245.0]
+  <cand> 4 at token 6705 sub_idx=6 pixel=[187.0,131.0]
+  <cand> 5 at token 6753 sub_idx=6 pixel=[192.0,200.0]
+  <cand> 6 at token 6799 sub_idx=6 pixel=[152.0,263.0]
+  <cand> 7 at token 6847 sub_idx=6 pixel=[355.0,303.0]
+  <cand> 8 at token 6895 sub_idx=6 pixel=[226.0,199.0]
+  <cand> 9 at token 6943 sub_idx=6 pixel=[356.0,288.0]
+  <cand> 10 at token 6991 sub_idx=6 pixel=[260.0,234.0]
+  <cand> 11 at token 7040 sub_idx=6 pixel=[203.0,143.0]
+  <cand> 12 at token 7089 sub_idx=6 pixel=[156.0,230.0]
+  <cand> 13 at token 7138 sub_idx=6 pixel=[286.0,277.0]
+  <cand> 14 at token 7187 sub_idx=6 pixel=[178.0,270.0]
+  <cand> 0 at token 7702 sub_idx=7 pixel=[249.0,247.0]
+  <cand> 1 at token 7748 sub_idx=7 pixel=[119.0,208.0]
+  <cand> 2 at token 7796 sub_idx=7 pixel=[59.0,192.0]
+  <cand> 3 at token 7844 sub_idx=7 pixel=[264.0,250.0]
+  <cand> 4 at token 7892 sub_idx=7 pixel=[263.0,200.0]
+  <cand> 5 at token 7940 sub_idx=7 pixel=[171.0,134.0]
+  <cand> 6 at token 7988 sub_idx=7 pixel=[215.0,260.0]
+  <cand> 7 at token 8036 sub_idx=7 pixel=[280.0,241.0]
+  <cand> 8 at token 8084 sub_idx=7 pixel=[180.0,110.0]
+  <cand> 0 at token 8600 sub_idx=8 pixel=[7.0,152.0]
+  <cand> 1 at token 8648 sub_idx=8 pixel=[114.0,86.0]
+  <cand> 2 at token 8696 sub_idx=8 pixel=[71.0,289.0]
+  <cand> 3 at token 8744 sub_idx=8 pixel=[5.0,218.0]
+  <cand> 4 at token 8792 sub_idx=8 pixel=[153.0,313.0]
+  <cand> 5 at token 8840 sub_idx=8 pixel=[203.0,321.0]
+  <cand> 6 at token 8888 sub_idx=8 pixel=[39.0,235.0]
+  <cand> 7 at token 8936 sub_idx=8 pixel=[232.0,227.0]
+  <cand> 8 at token 8982 sub_idx=8 pixel=[35.0,127.0]
+  <cand> 9 at token 9030 sub_idx=8 pixel=[95.0,119.0]
+  <cand> 10 at token 9078 sub_idx=8 pixel=[150.0,172.0]
+  <cand> 11 at token 9127 sub_idx=8 pixel=[226.0,243.0]
+  <cand> 0 at token 9642 sub_idx=9 pixel=[244.0,329.0]
+  <cand> 1 at token 9690 sub_idx=9 pixel=[226.0,285.0]
+  <cand> 2 at token 9738 sub_idx=9 pixel=[172.0,212.0]
+  <cand> 0 at token 10252 sub_idx=10 pixel=[250.0,304.0]
+  <cand> 1 at token 10300 sub_idx=10 pixel=[251.0,104.0]
+  <cand> 2 at token 10348 sub_idx=10 pixel=[213.0,270.0]
+  <cand> 3 at token 10396 sub_idx=10 pixel=[233.0,324.0]
+  <cand> 4 at token 10444 sub_idx=10 pixel=[217.0,3.0]
+  <cand> 0 at token 10957 sub_idx=11 pixel=[37.0,296.0]
+  <cand> 1 at token 11005 sub_idx=11 pixel=[30.0,314.0]
+  <cand> 2 at token 11053 sub_idx=11 pixel=[127.0,317.0]
+  <cand> 3 at token 11101 sub_idx=11 pixel=[22.0,286.0]
+  <cand> 4 at token 11149 sub_idx=11 pixel=[197.0,227.0]
+  <cand> 5 at token 11197 sub_idx=11 pixel=[314.0,262.0]
+  <cand> 6 at token 11245 sub_idx=11 pixel=[260.0,282.0]
+  <cand> 7 at token 11293 sub_idx=11 pixel=[314.0,171.0]
+  <cand> 8 at token 11339 sub_idx=11 pixel=[296.0,187.0]
+  <cand> 9 at token 11387 sub_idx=11 pixel=[234.0,269.0]
+  <cand> 10 at token 11435 sub_idx=11 pixel=[212.0,161.0]
+  <cand> 11 at token 11484 sub_idx=11 pixel=[322.0,153.0]
+  <cand> 12 at token 11533 sub_idx=11 pixel=[57.0,299.0]
+  <cand> 13 at token 11582 sub_idx=11 pixel=[301.0,151.0]
+  <cand> 14 at token 11631 sub_idx=11 pixel=[67.0,281.0]
+  <cand> 15 at token 11680 sub_idx=11 pixel=[60.0,284.0]
+  <cand> 16 at token 11729 sub_idx=11 pixel=[177.0,199.0]
+  <cand> 17 at token 11778 sub_idx=11 pixel=[50.0,324.0]
+  <cand> 18 at token 11827 sub_idx=11 pixel=[156.0,309.0]
+  <cand> 0 at token 12341 sub_idx=12 pixel=[270.0,120.0]
+  <cand> 1 at token 12389 sub_idx=12 pixel=[218.0,355.0]
+  <cand> 2 at token 12437 sub_idx=12 pixel=[301.0,310.0]
+  <cand> 3 at token 12485 sub_idx=12 pixel=[222.0,276.0]
+  <cand> 4 at token 12533 sub_idx=12 pixel=[148.0,143.0]
+  <cand> 5 at token 12581 sub_idx=12 pixel=[293.0,269.0]
+  <cand> 6 at token 12629 sub_idx=12 pixel=[199.0,357.0]
+  <cand> 7 at token 12677 sub_idx=12 pixel=[252.0,92.0]
+  <cand> 8 at token 12723 sub_idx=12 pixel=[256.0,172.0]
+  <cand> 9 at token 12771 sub_idx=12 pixel=[191.0,152.0]
+  <cand> 10 at token 12819 sub_idx=12 pixel=[166.0,123.0]
+  <cand> 11 at token 12868 sub_idx=12 pixel=[258.0,276.0]
+  <cand> 0 at token 13383 sub_idx=13 pixel=[116.0,124.0]
+  <cand> 1 at token 13431 sub_idx=13 pixel=[160.0,139.0]
+  <cand> 2 at token 13479 sub_idx=13 pixel=[238.0,303.0]
+  <cand> 3 at token 13527 sub_idx=13 pixel=[163.0,257.0]
+  <cand> 4 at token 13575 sub_idx=13 pixel=[183.0,213.0]
+  <cand> 5 at token 13623 sub_idx=13 pixel=[137.0,148.0]
+  <cand> 6 at token 13671 sub_idx=13 pixel=[128.0,114.0]
+  <cand> 7 at token 13719 sub_idx=13 pixel=[161.0,98.0]
+  <cand> 8 at token 13767 sub_idx=13 pixel=[268.0,205.0]
+  <cand> 9 at token 13815 sub_idx=13 pixel=[131.0,107.0]
+  <cand> 10 at token 13863 sub_idx=13 pixel=[194.0,277.0]
+  <cand> 11 at token 13912 sub_idx=13 pixel=[236.0,335.0]
+  <cand> 12 at token 13961 sub_idx=13 pixel=[176.0,281.0]
+  <cand> 0 at token 14475 sub_idx=14 pixel=[341.0,209.0]
+  <cand> 1 at token 14523 sub_idx=14 pixel=[208.0,231.0]
+  <cand> 2 at token 14571 sub_idx=14 pixel=[211.0,191.0]
+  <cand> 3 at token 14619 sub_idx=14 pixel=[231.0,187.0]
+  <cand> 4 at token 14667 sub_idx=14 pixel=[295.0,235.0]
+  <cand> 5 at token 14715 sub_idx=14 pixel=[274.0,113.0]
+  <cand> 6 at token 14763 sub_idx=14 pixel=[278.0,137.0]
+  <cand> 7 at token 14811 sub_idx=14 pixel=[330.0,162.0]
+  <cand> 8 at token 14859 sub_idx=14 pixel=[307.0,144.0]
+  <cand> 9 at token 14907 sub_idx=14 pixel=[318.0,181.0]
+  <cand> 0 at token 15421 sub_idx=15 pixel=[269.0,223.0]
+  <cand> 1 at token 15469 sub_idx=15 pixel=[204.0,178.0]
+  <cand> 2 at token 15517 sub_idx=15 pixel=[213.0,197.0]
+  <cand> 0 at token 16030 sub_idx=16 pixel=[336.0,162.0]
+  <cand> 1 at token 16078 sub_idx=16 pixel=[264.0,117.0]
+  <cand> 2 at token 16126 sub_idx=16 pixel=[355.0,214.0]
+  <cand> 3 at token 16174 sub_idx=16 pixel=[327.0,150.0]
+  <cand> 4 at token 16222 sub_idx=16 pixel=[156.0,161.0]
+  <cand> 5 at token 16270 sub_idx=16 pixel=[420.0,167.0]
+  <cand> 6 at token 16318 sub_idx=16 pixel=[147.0,187.0]
+  <cand> 7 at token 16366 sub_idx=16 pixel=[148.0,203.0]
+  <cand> 8 at token 16414 sub_idx=16 pixel=[239.0,145.0]
+  <cand> 9 at token 16462 sub_idx=16 pixel=[424.0,92.0]
+  <cand> 10 at token 16510 sub_idx=16 pixel=[214.0,290.0]
+  <cand> 11 at token 16559 sub_idx=16 pixel=[196.0,316.0]
+  <cand> 12 at token 16608 sub_idx=16 pixel=[195.0,261.0]
+  <cand> 13 at token 16657 sub_idx=16 pixel=[330.0,213.0]
+  <cand> 14 at token 16706 sub_idx=16 pixel=[247.0,230.0]
+  <cand> 15 at token 16755 sub_idx=16 pixel=[178.0,293.0]
+  <cand> 16 at token 16804 sub_idx=16 pixel=[156.0,168.0]
+  <cand> 17 at token 16853 sub_idx=16 pixel=[286.0,121.0]
+  <cand> 18 at token 16902 sub_idx=16 pixel=[251.0,317.0]
+  <cand> 19 at token 16951 sub_idx=16 pixel=[167.0,139.0]
+  <cand> 20 at token 17000 sub_idx=16 pixel=[409.0,99.0]
+  <cand> 0 at token 17515 sub_idx=17 pixel=[228.0,247.0]
+  <cand> 1 at token 17563 sub_idx=17 pixel=[288.0,254.0]
+  <cand> 2 at token 17611 sub_idx=17 pixel=[220.0,206.0]
+  <cand> 3 at token 17659 sub_idx=17 pixel=[221.0,334.0]
+  <cand> 0 at token 18175 sub_idx=18 pixel=[224.0,119.0]
+  <cand> 1 at token 18223 sub_idx=18 pixel=[406.0,250.0]
+  <cand> 2 at token 18271 sub_idx=18 pixel=[358.0,253.0]
+  <cand> 3 at token 18319 sub_idx=18 pixel=[348.0,228.0]
+  <cand> 4 at token 18367 sub_idx=18 pixel=[244.0,150.0]
+  <cand> 5 at token 18415 sub_idx=18 pixel=[195.0,207.0]
+  <cand> 6 at token 18463 sub_idx=18 pixel=[250.0,138.0]
+  <cand> 7 at token 18511 sub_idx=18 pixel=[417.0,106.0]
+  <cand> 8 at token 18559 sub_idx=18 pixel=[226.0,177.0]
+  <cand> 9 at token 18607 sub_idx=18 pixel=[399.0,289.0]
+  <cand> 10 at token 18655 sub_idx=18 pixel=[415.0,149.0]
+  <cand> 11 at token 18704 sub_idx=18 pixel=[209.0,271.0]
+  <cand> 12 at token 18751 sub_idx=18 pixel=[428.0,276.0]
+  <cand> 13 at token 18800 sub_idx=18 pixel=[403.0,169.0]
+  <cand> 0 at token 19316 sub_idx=19 pixel=[241.0,242.0]
+  <cand> 1 at token 19364 sub_idx=19 pixel=[270.0,236.0]
+  <cand> 2 at token 19412 sub_idx=19 pixel=[222.0,274.0]
+  <cand> 3 at token 19460 sub_idx=19 pixel=[292.0,243.0]
+  <cand> 4 at token 19508 sub_idx=19 pixel=[258.0,75.0]
+  <cand> 5 at token 19556 sub_idx=19 pixel=[202.0,232.0]
+  <cand> 6 at token 19604 sub_idx=19 pixel=[229.0,89.0]
+  <cand> 7 at token 19652 sub_idx=19 pixel=[244.0,225.0]
+  <cand> 8 at token 19700 sub_idx=19 pixel=[247.0,103.0]
+  <cand> 9 at token 19748 sub_idx=19 pixel=[190.0,276.0]
+  <cand> 0 at token 20261 sub_idx=20 pixel=[131.0,352.0]
+  <cand> 1 at token 20309 sub_idx=20 pixel=[186.0,236.0]
+  <cand> 2 at token 20357 sub_idx=20 pixel=[228.0,118.0]
+  <cand> 3 at token 20405 sub_idx=20 pixel=[283.0,238.0]
+  <cand> 4 at token 20453 sub_idx=20 pixel=[162.0,256.0]
+  <cand> 5 at token 20501 sub_idx=20 pixel=[153.0,340.0]
+  <cand> 6 at token 20549 sub_idx=20 pixel=[170.0,237.0]
+  <cand> 0 at token 21062 sub_idx=21 pixel=[214.0,247.0]
+  <cand> 1 at token 21110 sub_idx=21 pixel=[174.0,242.0]
+  <cand> 2 at token 21158 sub_idx=21 pixel=[221.0,302.0]
+  <cand> 3 at token 21206 sub_idx=21 pixel=[175.0,257.0]
+  <cand> 4 at token 21254 sub_idx=21 pixel=[268.0,238.0]
+  <cand> 5 at token 21302 sub_idx=21 pixel=[294.0,212.0]
+  <cand> 6 at token 21350 sub_idx=21 pixel=[214.0,268.0]
+  <cand> 7 at token 21398 sub_idx=21 pixel=[297.0,236.0]
+  <cand> 8 at token 21446 sub_idx=21 pixel=[234.0,261.0]
+  <cand> 9 at token 21494 sub_idx=21 pixel=[197.0,249.0]
+  <cand> 0 at token 22008 sub_idx=22 pixel=[138.0,189.0]
+  <cand> 1 at token 22056 sub_idx=22 pixel=[106.0,189.0]
+  <cand> 2 at token 22104 sub_idx=22 pixel=[234.0,164.0]
+  <cand> 3 at token 22152 sub_idx=22 pixel=[291.0,224.0]
+  <cand> 4 at token 22200 sub_idx=22 pixel=[107.0,149.0]
+  <cand> 5 at token 22248 sub_idx=22 pixel=[178.0,153.0]
+  <cand> 6 at token 22296 sub_idx=22 pixel=[126.0,200.0]
+  <cand> 7 at token 22344 sub_idx=22 pixel=[128.0,180.0]
+  <cand> 8 at token 22392 sub_idx=22 pixel=[251.0,202.0]
+  <cand> 9 at token 22438 sub_idx=22 pixel=[21.0,273.0]
+  <cand> 10 at token 22486 sub_idx=22 pixel=[83.0,194.0]
+  <cand> 11 at token 22535 sub_idx=22 pixel=[89.0,240.0]
+  <cand> 12 at token 22584 sub_idx=22 pixel=[49.0,253.0]
+  <cand> 13 at token 22633 sub_idx=22 pixel=[203.0,207.0]
+  <cand> 14 at token 22682 sub_idx=22 pixel=[96.0,148.0]
+  <cand> 15 at token 22731 sub_idx=22 pixel=[262.0,153.0]
+  <cand> 16 at token 22780 sub_idx=22 pixel=[86.0,151.0]
+  <cand> 17 at token 22829 sub_idx=22 pixel=[85.0,169.0]
+  <cand> 0 at token 23343 sub_idx=23 pixel=[251.0,113.0]
+  <cand> 1 at token 23391 sub_idx=23 pixel=[221.0,124.0]
+  <cand> 2 at token 23439 sub_idx=23 pixel=[244.0,150.0]
+  <cand> 3 at token 23487 sub_idx=23 pixel=[303.0,214.0]
+  <cand> 0 at token 24000 sub_idx=24 pixel=[266.0,166.0]
+  <cand> 1 at token 24048 sub_idx=24 pixel=[200.0,101.0]
+  <cand> 2 at token 24096 sub_idx=24 pixel=[260.0,121.0]
+  <e_cand> 0 at token 490 sub_idx=0 (same embed)
+  <e_cand> 1 at token 538 sub_idx=0 (same embed)
+  <e_cand> 2 at token 586 sub_idx=0 (same embed)
+  <e_cand> 0 at token 1387 sub_idx=1 (same embed)
+  <e_cand> 1 at token 1435 sub_idx=1 (same embed)
+  <e_cand> 2 at token 1483 sub_idx=1 (same embed)
+  <e_cand> 0 at token 2527 sub_idx=2 (same embed)
+  <e_cand> 1 at token 2575 sub_idx=2 (same embed)
+  <e_cand> 2 at token 2623 sub_idx=2 (same embed)
+  <e_cand> 0 at token 3522 sub_idx=3 (same embed)
+  <e_cand> 1 at token 3570 sub_idx=3 (same embed)
+  <e_cand> 2 at token 3618 sub_idx=3 (same embed)
+  <e_cand> 0 at token 4516 sub_idx=4 (same embed)
+  <e_cand> 1 at token 4564 sub_idx=4 (same embed)
+  <e_cand> 2 at token 4612 sub_idx=4 (same embed)
+  <e_cand> 0 at token 5709 sub_idx=5 (same embed)
+  <e_cand> 1 at token 5757 sub_idx=5 (same embed)
+  <e_cand> 2 at token 5805 sub_idx=5 (same embed)
+  <e_cand> 0 at token 6559 sub_idx=6 (same embed)
+  <e_cand> 1 at token 6607 sub_idx=6 (same embed)
+  <e_cand> 2 at token 6655 sub_idx=6 (same embed)
+  <e_cand> 0 at token 7746 sub_idx=7 (same embed)
+  <e_cand> 1 at token 7794 sub_idx=7 (same embed)
+  <e_cand> 2 at token 7842 sub_idx=7 (same embed)
+  <e_cand> 0 at token 8646 sub_idx=8 (same embed)
+  <e_cand> 1 at token 8694 sub_idx=8 (same embed)
+  <e_cand> 2 at token 8742 sub_idx=8 (same embed)
+  <e_cand> 0 at token 9688 sub_idx=9 (same embed)
+  <e_cand> 1 at token 9736 sub_idx=9 (same embed)
+  <e_cand> 2 at token 9784 sub_idx=9 (same embed)
+  <e_cand> 0 at token 10298 sub_idx=10 (same embed)
+  <e_cand> 1 at token 10346 sub_idx=10 (same embed)
+  <e_cand> 2 at token 10394 sub_idx=10 (same embed)
+  <e_cand> 0 at token 11003 sub_idx=11 (same embed)
+  <e_cand> 1 at token 11051 sub_idx=11 (same embed)
+  <e_cand> 2 at token 11099 sub_idx=11 (same embed)
+  <e_cand> 0 at token 12387 sub_idx=12 (same embed)
+  <e_cand> 1 at token 12435 sub_idx=12 (same embed)
+  <e_cand> 2 at token 12483 sub_idx=12 (same embed)
+  <e_cand> 0 at token 13429 sub_idx=13 (same embed)
+  <e_cand> 1 at token 13477 sub_idx=13 (same embed)
+  <e_cand> 2 at token 13525 sub_idx=13 (same embed)
+  <e_cand> 0 at token 14521 sub_idx=14 (same embed)
+  <e_cand> 1 at token 14569 sub_idx=14 (same embed)
+  <e_cand> 2 at token 14617 sub_idx=14 (same embed)
+  <e_cand> 0 at token 15467 sub_idx=15 (same embed)
+  <e_cand> 1 at token 15515 sub_idx=15 (same embed)
+  <e_cand> 2 at token 15563 sub_idx=15 (same embed)
+  <e_cand> 0 at token 16076 sub_idx=16 (same embed)
+  <e_cand> 1 at token 16124 sub_idx=16 (same embed)
+  <e_cand> 2 at token 16172 sub_idx=16 (same embed)
+  <e_cand> 0 at token 17561 sub_idx=17 (same embed)
+  <e_cand> 1 at token 17609 sub_idx=17 (same embed)
+  <e_cand> 2 at token 17657 sub_idx=17 (same embed)
+  <e_cand> 0 at token 18221 sub_idx=18 (same embed)
+  <e_cand> 1 at token 18269 sub_idx=18 (same embed)
+  <e_cand> 2 at token 18317 sub_idx=18 (same embed)
+  <e_cand> 0 at token 19362 sub_idx=19 (same embed)
+  <e_cand> 1 at token 19410 sub_idx=19 (same embed)
+  <e_cand> 2 at token 19458 sub_idx=19 (same embed)
+  <e_cand> 0 at token 20307 sub_idx=20 (same embed)
+  <e_cand> 1 at token 20355 sub_idx=20 (same embed)
+  <e_cand> 2 at token 20403 sub_idx=20 (same embed)
+  <e_cand> 0 at token 21108 sub_idx=21 (same embed)
+  <e_cand> 1 at token 21156 sub_idx=21 (same embed)
+  <e_cand> 2 at token 21204 sub_idx=21 (same embed)
+  <e_cand> 0 at token 22054 sub_idx=22 (same embed)
+  <e_cand> 1 at token 22102 sub_idx=22 (same embed)
+  <e_cand> 2 at token 22150 sub_idx=22 (same embed)
+  <e_cand> 0 at token 23389 sub_idx=23 (same embed)
+  <e_cand> 1 at token 23437 sub_idx=23 (same embed)
+  <e_cand> 2 at token 23485 sub_idx=23 (same embed)
+  <e_cand> 0 at token 24046 sub_idx=24 (same embed)
+  <e_cand> 1 at token 24094 sub_idx=24 (same embed)
+  <e_cand> 2 at token 24142 sub_idx=24 (same embed)
+
+[2026-02-25 06:18:02,804] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 2.25 | optimizer_gradients: 5.18 | optimizer_step: 3.28
+[2026-02-25 06:18:02,815] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6729.83 | bwd_microstep: 12622.88 | bwd_inner_microstep: 6539.62 | bwd_allreduce_microstep: 6083.17 | step_microstep: 47.45
+[2026-02-25 06:18:02,816] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6729.83 | bwd: 12622.88 | bwd_inner: 6539.61 | bwd_allreduce: 6083.19 | step: 48.26
+{'loss': 1.6327, 'learning_rate': 6.060606060606061e-07, 'epoch': 0.0}
+02/25/2026 06:18:09 - INFO - internvl_cleaned.model.internvl_chat.modeling_internvl_chat - [Text Position Embedding Verification] Sample 3:
+  <s> at token 437 sub_idx=0
+  <s> at token 1145 sub_idx=1
+  <s> at token 2237 sub_idx=2
+  <s> at token 3038 sub_idx=3
+  <s> at token 3888 sub_idx=4
+  <s> at token 5374 sub_idx=5
+  <s> at token 6370 sub_idx=6
+  <s> at token 7411 sub_idx=7
+  <s> at token 8261 sub_idx=8
+  <s> at token 9254 sub_idx=9
+  <s> at token 10887 sub_idx=10
+  <s> at token 12126 sub_idx=11
+  <s> at token 13072 sub_idx=12
+  <s> at token 13967 sub_idx=13
+  <s> at token 14913 sub_idx=14
+  <s> at token 15811 sub_idx=15
+  <s> at token 16658 sub_idx=16
+  <s> at token 17459 sub_idx=17
+  <s> at token 18500 sub_idx=18
+  <s> at token 19492 sub_idx=19
+  <s> at token 20295 sub_idx=20
+  <s> at token 21049 sub_idx=21
+  <s> at token 21803 sub_idx=22
+  <s> at token 22750 sub_idx=23
+  <s> at token 23598 sub_idx=24
+  <s> at token 24448 sub_idx=25
+  <e_s> at token 439 sub_idx=0
+  <e_s> at token 1147 sub_idx=1
+  <e_s> at token 2239 sub_idx=2
+  <e_s> at token 3040 sub_idx=3
+  <e_s> at token 3890 sub_idx=4
+  <e_s> at token 5376 sub_idx=5
+  <e_s> at token 6372 sub_idx=6
+  <e_s> at token 7413 sub_idx=7
+  <e_s> at token 8263 sub_idx=8
+  <e_s> at token 9256 sub_idx=9
+  <e_s> at token 10889 sub_idx=10
+  <e_s> at token 12128 sub_idx=11
+  <e_s> at token 13074 sub_idx=12
+  <e_s> at token 13969 sub_idx=13
+  <e_s> at token 14915 sub_idx=14
+  <e_s> at token 15813 sub_idx=15
+  <e_s> at token 16660 sub_idx=16
+  <e_s> at token 17461 sub_idx=17
+  <e_s> at token 18502 sub_idx=18
+  <e_s> at token 19494 sub_idx=19
+  <e_s> at token 20297 sub_idx=20
+  <e_s> at token 21051 sub_idx=21
+  <e_s> at token 21805 sub_idx=22
+  <e_s> at token 22752 sub_idx=23
+  <e_s> at token 23600 sub_idx=24
+  <e_s> at token 24450 sub_idx=25
+  <cand> 0 at token 444 sub_idx=0 pixel=[229.0,302.0]
+  <cand> 1 at token 492 sub_idx=0 pixel=[152.0,186.0]
+  <cand> 2 at token 540 sub_idx=0 pixel=[122.0,189.0]
+  <cand> 3 at token 588 sub_idx=0 pixel=[155.0,209.0]
+  <cand> 4 at token 636 sub_idx=0 pixel=[189.0,226.0]
+  <cand> 0 at token 1152 sub_idx=1 pixel=[170.0,104.0]
+  <cand> 1 at token 1200 sub_idx=1 pixel=[192.0,94.0]
+  <cand> 2 at token 1248 sub_idx=1 pixel=[181.0,77.0]
+  <cand> 3 at token 1296 sub_idx=1 pixel=[248.0,199.0]
+  <cand> 4 at token 1344 sub_idx=1 pixel=[243.0,272.0]
+  <cand> 5 at token 1392 sub_idx=1 pixel=[288.0,95.0]
+  <cand> 6 at token 1440 sub_idx=1 pixel=[242.0,239.0]
+  <cand> 7 at token 1488 sub_idx=1 pixel=[186.0,256.0]
+  <cand> 8 at token 1536 sub_idx=1 pixel=[272.0,134.0]
+  <cand> 9 at token 1584 sub_idx=1 pixel=[165.0,127.0]
+  <cand> 10 at token 1632 sub_idx=1 pixel=[214.0,59.0]
+  <cand> 11 at token 1681 sub_idx=1 pixel=[186.0,272.0]
+  <cand> 12 at token 1728 sub_idx=1 pixel=[148.0,129.0]
+  <cand> 0 at token 2244 sub_idx=2 pixel=[310.0,277.0]
+  <cand> 1 at token 2292 sub_idx=2 pixel=[256.0,155.0]
+  <cand> 2 at token 2340 sub_idx=2 pixel=[293.0,278.0]
+  <cand> 3 at token 2388 sub_idx=2 pixel=[304.0,258.0]
+  <cand> 4 at token 2436 sub_idx=2 pixel=[194.0,257.0]
+  <cand> 5 at token 2484 sub_idx=2 pixel=[198.0,186.0]
+  <cand> 6 at token 2532 sub_idx=2 pixel=[243.0,165.0]
+  <cand> 0 at token 3045 sub_idx=3 pixel=[313.0,271.0]
+  <cand> 1 at token 3093 sub_idx=3 pixel=[304.0,294.0]
+  <cand> 2 at token 3141 sub_idx=3 pixel=[287.0,231.0]
+  <cand> 3 at token 3189 sub_idx=3 pixel=[274.0,104.0]
+  <cand> 4 at token 3237 sub_idx=3 pixel=[116.0,235.0]
+  <cand> 5 at token 3285 sub_idx=3 pixel=[269.0,245.0]
+  <cand> 6 at token 3333 sub_idx=3 pixel=[161.0,212.0]
+  <cand> 7 at token 3381 sub_idx=3 pixel=[195.0,247.0]
+  <cand> 0 at token 3895 sub_idx=4 pixel=[89.0,206.0]
+  <cand> 1 at token 3943 sub_idx=4 pixel=[303.0,296.0]
+  <cand> 2 at token 3991 sub_idx=4 pixel=[258.0,208.0]
+  <cand> 3 at token 4039 sub_idx=4 pixel=[111.0,226.0]
+  <cand> 4 at token 4087 sub_idx=4 pixel=[123.0,269.0]
+  <cand> 5 at token 4135 sub_idx=4 pixel=[168.0,277.0]
+  <cand> 6 at token 4183 sub_idx=4 pixel=[229.0,266.0]
+  <cand> 7 at token 4231 sub_idx=4 pixel=[102.0,248.0]
+  <cand> 8 at token 4279 sub_idx=4 pixel=[279.0,378.0]
+  <cand> 9 at token 4327 sub_idx=4 pixel=[230.0,369.0]
+  <cand> 10 at token 4375 sub_idx=4 pixel=[249.0,349.0]
+  <cand> 11 at token 4424 sub_idx=4 pixel=[249.0,128.0]
+  <cand> 12 at token 4473 sub_idx=4 pixel=[260.0,284.0]
+  <cand> 13 at token 4522 sub_idx=4 pixel=[241.0,109.0]
+  <cand> 14 at token 4571 sub_idx=4 pixel=[140.0,243.0]
+  <cand> 15 at token 4620 sub_idx=4 pixel=[299.0,272.0]
+  <cand> 16 at token 4669 sub_idx=4 pixel=[112.0,187.0]
+  <cand> 17 at token 4718 sub_idx=4 pixel=[161.0,346.0]
+  <cand> 18 at token 4767 sub_idx=4 pixel=[173.0,210.0]
+  <cand> 19 at token 4816 sub_idx=4 pixel=[122.0,345.0]
+  <cand> 20 at token 4865 sub_idx=4 pixel=[257.0,355.0]
+  <cand> 0 at token 5381 sub_idx=5 pixel=[193.0,254.0]
+  <cand> 1 at token 5429 sub_idx=5 pixel=[201.0,319.0]
+  <cand> 2 at token 5477 sub_idx=5 pixel=[135.0,302.0]
+  <cand> 3 at token 5525 sub_idx=5 pixel=[200.0,241.0]
+  <cand> 4 at token 5573 sub_idx=5 pixel=[221.0,294.0]
+  <cand> 5 at token 5621 sub_idx=5 pixel=[187.0,272.0]
+  <cand> 6 at token 5669 sub_idx=5 pixel=[257.0,313.0]
+  <cand> 7 at token 5717 sub_idx=5 pixel=[237.0,250.0]
+  <cand> 8 at token 5765 sub_idx=5 pixel=[152.0,103.0]
+  <cand> 9 at token 5813 sub_idx=5 pixel=[246.0,281.0]
+  <cand> 10 at token 5861 sub_idx=5 pixel=[204.0,330.0]
+  <cand> 0 at token 6377 sub_idx=6 pixel=[119.0,307.0]
+  <cand> 1 at token 6425 sub_idx=6 pixel=[242.0,189.0]
+  <cand> 2 at token 6473 sub_idx=6 pixel=[272.0,272.0]
+  <cand> 3 at token 6521 sub_idx=6 pixel=[216.0,188.0]
+  <cand> 4 at token 6567 sub_idx=6 pixel=[110.0,294.0]
+  <cand> 5 at token 6615 sub_idx=6 pixel=[307.0,290.0]
+  <cand> 6 at token 6663 sub_idx=6 pixel=[120.0,326.0]
+  <cand> 7 at token 6711 sub_idx=6 pixel=[320.0,310.0]
+  <cand> 8 at token 6759 sub_idx=6 pixel=[231.0,183.0]
+  <cand> 9 at token 6807 sub_idx=6 pixel=[173.0,176.0]
+  <cand> 10 at token 6855 sub_idx=6 pixel=[207.0,203.0]
+  <cand> 11 at token 6904 sub_idx=6 pixel=[272.0,321.0]
+  <cand> 0 at token 7418 sub_idx=7 pixel=[246.0,234.0]
+  <cand> 1 at token 7464 sub_idx=7 pixel=[229.0,295.0]
+  <cand> 2 at token 7512 sub_idx=7 pixel=[176.0,318.0]
+  <cand> 3 at token 7560 sub_idx=7 pixel=[193.0,203.0]
+  <cand> 4 at token 7608 sub_idx=7 pixel=[135.0,197.0]
+  <cand> 5 at token 7656 sub_idx=7 pixel=[237.0,65.0]
+  <cand> 6 at token 7704 sub_idx=7 pixel=[253.0,249.0]
+  <cand> 7 at token 7752 sub_idx=7 pixel=[243.0,256.0]
+  <cand> 0 at token 8268 sub_idx=8 pixel=[193.0,211.0]
+  <cand> 1 at token 8316 sub_idx=8 pixel=[299.0,179.0]
+  <cand> 2 at token 8364 sub_idx=8 pixel=[323.0,190.0]
+  <cand> 3 at token 8412 sub_idx=8 pixel=[206.0,270.0]
+  <cand> 4 at token 8458 sub_idx=8 pixel=[202.0,88.0]
+  <cand> 5 at token 8506 sub_idx=8 pixel=[254.0,142.0]
+  <cand> 6 at token 8554 sub_idx=8 pixel=[360.0,198.0]
+  <cand> 7 at token 8602 sub_idx=8 pixel=[222.0,125.0]
+  <cand> 8 at token 8650 sub_idx=8 pixel=[219.0,97.0]
+  <cand> 9 at token 8698 sub_idx=8 pixel=[349.0,215.0]
+  <cand> 10 at token 8746 sub_idx=8 pixel=[308.0,112.0]
+  <cand> 0 at token 9261 sub_idx=9 pixel=[155.0,287.0]
+  <cand> 1 at token 9309 sub_idx=9 pixel=[195.0,291.0]
+  <cand> 2 at token 9357 sub_idx=9 pixel=[164.0,298.0]
+  <cand> 3 at token 9405 sub_idx=9 pixel=[257.0,123.0]
+  <cand> 4 at token 9453 sub_idx=9 pixel=[268.0,152.0]
+  <cand> 5 at token 9501 sub_idx=9 pixel=[301.0,179.0]
+  <cand> 6 at token 9549 sub_idx=9 pixel=[348.0,323.0]
+  <cand> 7 at token 9597 sub_idx=9 pixel=[192.0,128.0]
+  <cand> 8 at token 9645 sub_idx=9 pixel=[230.0,285.0]
+  <cand> 9 at token 9693 sub_idx=9 pixel=[322.0,200.0]
+  <cand> 10 at token 9741 sub_idx=9 pixel=[229.0,243.0]
+  <cand> 11 at token 9788 sub_idx=9 pixel=[376.0,261.0]
+  <cand> 12 at token 9837 sub_idx=9 pixel=[302.0,195.0]
+  <cand> 13 at token 9886 sub_idx=9 pixel=[274.0,368.0]
+  <cand> 14 at token 9935 sub_idx=9 pixel=[372.0,364.0]
+  <cand> 15 at token 9984 sub_idx=9 pixel=[342.0,173.0]
+  <cand> 16 at token 10033 sub_idx=9 pixel=[311.0,303.0]
+  <cand> 17 at token 10082 sub_idx=9 pixel=[376.0,344.0]
+  <cand> 18 at token 10131 sub_idx=9 pixel=[222.0,144.0]
+  <cand> 19 at token 10180 sub_idx=9 pixel=[394.0,290.0]
+  <cand> 20 at token 10229 sub_idx=9 pixel=[212.0,252.0]
+  <cand> 21 at token 10278 sub_idx=9 pixel=[330.0,349.0]
+  <cand> 22 at token 10327 sub_idx=9 pixel=[239.0,143.0]
+  <cand> 23 at token 10376 sub_idx=9 pixel=[344.0,300.0]
+  <cand> 0 at token 10894 sub_idx=10 pixel=[297.0,230.0]
+  <cand> 1 at token 10942 sub_idx=10 pixel=[215.0,329.0]
+  <cand> 2 at token 10990 sub_idx=10 pixel=[176.0,245.0]
+  <cand> 3 at token 11038 sub_idx=10 pixel=[209.0,170.0]
+  <cand> 4 at token 11086 sub_idx=10 pixel=[322.0,354.0]
+  <cand> 5 at token 11134 sub_idx=10 pixel=[156.0,177.0]
+  <cand> 6 at token 11182 sub_idx=10 pixel=[280.0,202.0]
+  <cand> 7 at token 11230 sub_idx=10 pixel=[136.0,247.0]
+  <cand> 8 at token 11278 sub_idx=10 pixel=[375.0,363.0]
+  <cand> 9 at token 11326 sub_idx=10 pixel=[262.0,159.0]
+  <cand> 10 at token 11374 sub_idx=10 pixel=[282.0,121.0]
+  <cand> 11 at token 11423 sub_idx=10 pixel=[166.0,196.0]
+  <cand> 12 at token 11472 sub_idx=10 pixel=[296.0,109.0]
+  <cand> 13 at token 11521 sub_idx=10 pixel=[292.0,98.0]
+  <cand> 14 at token 11570 sub_idx=10 pixel=[154.0,218.0]
+  <cand> 15 at token 11619 sub_idx=10 pixel=[217.0,278.0]
+  <cand> 0 at token 12133 sub_idx=11 pixel=[257.0,140.0]
+  <cand> 1 at token 12181 sub_idx=11 pixel=[181.0,185.0]
+  <cand> 2 at token 12229 sub_idx=11 pixel=[287.0,274.0]
+  <cand> 3 at token 12277 sub_idx=11 pixel=[249.0,204.0]
+  <cand> 4 at token 12325 sub_idx=11 pixel=[230.0,152.0]
+  <cand> 5 at token 12373 sub_idx=11 pixel=[250.0,134.0]
+  <cand> 6 at token 12421 sub_idx=11 pixel=[288.0,222.0]
+  <cand> 7 at token 12469 sub_idx=11 pixel=[124.0,273.0]
+  <cand> 8 at token 12517 sub_idx=11 pixel=[346.0,232.0]
+  <cand> 9 at token 12565 sub_idx=11 pixel=[344.0,264.0]
+  <cand> 0 at token 13079 sub_idx=12 pixel=[67.0,162.0]
+  <cand> 1 at token 13127 sub_idx=12 pixel=[109.0,224.0]
+  <cand> 2 at token 13175 sub_idx=12 pixel=[231.0,230.0]
+  <cand> 3 at token 13221 sub_idx=12 pixel=[151.0,174.0]
+  <cand> 4 at token 13269 sub_idx=12 pixel=[73.0,164.0]
+  <cand> 5 at token 13317 sub_idx=12 pixel=[203.0,215.0]
+  <cand> 6 at token 13365 sub_idx=12 pixel=[89.0,165.0]
+  <cand> 7 at token 13413 sub_idx=12 pixel=[108.0,193.0]
+  <cand> 8 at token 13461 sub_idx=12 pixel=[135.0,230.0]
+  <cand> 0 at token 13974 sub_idx=13 pixel=[247.0,149.0]
+  <cand> 1 at token 14022 sub_idx=13 pixel=[265.0,271.0]
+  <cand> 2 at token 14070 sub_idx=13 pixel=[152.0,198.0]
+  <cand> 3 at token 14118 sub_idx=13 pixel=[182.0,202.0]
+  <cand> 4 at token 14166 sub_idx=13 pixel=[268.0,310.0]
+  <cand> 5 at token 14214 sub_idx=13 pixel=[237.0,209.0]
+  <cand> 6 at token 14262 sub_idx=13 pixel=[244.0,107.0]
+  <cand> 7 at token 14310 sub_idx=13 pixel=[198.0,223.0]
+  <cand> 8 at token 14358 sub_idx=13 pixel=[238.0,242.0]
+  <cand> 9 at token 14406 sub_idx=13 pixel=[220.0,124.0]
+  <cand> 0 at token 14920 sub_idx=14 pixel=[146.0,258.0]
+  <cand> 1 at token 14968 sub_idx=14 pixel=[239.0,149.0]
+  <cand> 2 at token 15016 sub_idx=14 pixel=[145.0,290.0]
+  <cand> 3 at token 15064 sub_idx=14 pixel=[289.0,247.0]
+  <cand> 4 at token 15112 sub_idx=14 pixel=[132.0,108.0]
+  <cand> 5 at token 15160 sub_idx=14 pixel=[267.0,221.0]
+  <cand> 6 at token 15208 sub_idx=14 pixel=[302.0,234.0]
+  <cand> 7 at token 15256 sub_idx=14 pixel=[239.0,204.0]
+  <cand> 8 at token 15304 sub_idx=14 pixel=[166.0,205.0]
+  <cand> 0 at token 15818 sub_idx=15 pixel=[232.0,165.0]
+  <cand> 1 at token 15866 sub_idx=15 pixel=[208.0,176.0]
+  <cand> 2 at token 15914 sub_idx=15 pixel=[230.0,142.0]
+  <cand> 3 at token 15962 sub_idx=15 pixel=[211.0,143.0]
+  <cand> 4 at token 16010 sub_idx=15 pixel=[227.0,131.0]
+  <cand> 5 at token 16056 sub_idx=15 pixel=[222.0,265.0]
+  <cand> 6 at token 16104 sub_idx=15 pixel=[194.0,268.0]
+  <cand> 7 at token 16152 sub_idx=15 pixel=[240.0,216.0]
+  <cand> 0 at token 16665 sub_idx=16 pixel=[254.0,190.0]
+  <cand> 1 at token 16713 sub_idx=16 pixel=[228.0,272.0]
+  <cand> 2 at token 16761 sub_idx=16 pixel=[229.0,191.0]
+  <cand> 3 at token 16809 sub_idx=16 pixel=[263.0,264.0]
+  <cand> 4 at token 16857 sub_idx=16 pixel=[221.0,318.0]
+  <cand> 5 at token 16905 sub_idx=16 pixel=[241.0,314.0]
+  <cand> 6 at token 16953 sub_idx=16 pixel=[261.0,283.0]
+  <cand> 0 at token 17466 sub_idx=17 pixel=[219.0,186.0]
+  <cand> 1 at token 17514 sub_idx=17 pixel=[212.0,228.0]
+  <cand> 2 at token 17560 sub_idx=17 pixel=[197.0,98.0]
+  <cand> 3 at token 17608 sub_idx=17 pixel=[293.0,185.0]
+  <cand> 4 at token 17656 sub_idx=17 pixel=[299.0,121.0]
+  <cand> 5 at token 17704 sub_idx=17 pixel=[208.0,270.0]
+  <cand> 6 at token 17752 sub_idx=17 pixel=[355.0,202.0]
+  <cand> 7 at token 17800 sub_idx=17 pixel=[218.0,137.0]
+  <cand> 8 at token 17848 sub_idx=17 pixel=[199.0,234.0]
+  <cand> 9 at token 17896 sub_idx=17 pixel=[347.0,218.0]
+  <cand> 10 at token 17944 sub_idx=17 pixel=[213.0,107.0]
+  <cand> 11 at token 17993 sub_idx=17 pixel=[254.0,143.0]
+  <cand> 0 at token 18507 sub_idx=18 pixel=[182.0,261.0]
+  <cand> 1 at token 18555 sub_idx=18 pixel=[147.0,252.0]
+  <cand> 2 at token 18603 sub_idx=18 pixel=[174.0,216.0]
+  <cand> 3 at token 18651 sub_idx=18 pixel=[191.0,237.0]
+  <cand> 4 at token 18699 sub_idx=18 pixel=[147.0,215.0]
+  <cand> 5 at token 18747 sub_idx=18 pixel=[269.0,230.0]
+  <cand> 6 at token 18793 sub_idx=18 pixel=[278.0,201.0]
+  <cand> 7 at token 18841 sub_idx=18 pixel=[281.0,227.0]
+  <cand> 8 at token 18889 sub_idx=18 pixel=[173.0,268.0]
+  <cand> 9 at token 18937 sub_idx=18 pixel=[252.0,230.0]
+  <cand> 10 at token 18985 sub_idx=18 pixel=[186.0,212.0]
+  <cand> 0 at token 19499 sub_idx=19 pixel=[185.0,30.0]
+  <cand> 1 at token 19547 sub_idx=19 pixel=[170.0,89.0]
+  <cand> 2 at token 19595 sub_idx=19 pixel=[176.0,303.0]
+  <cand> 3 at token 19643 sub_idx=19 pixel=[185.0,244.0]
+  <cand> 4 at token 19691 sub_idx=19 pixel=[272.0,184.0]
+  <cand> 5 at token 19739 sub_idx=19 pixel=[155.0,133.0]
+  <cand> 6 at token 19787 sub_idx=19 pixel=[258.0,102.0]
+  <cand> 0 at token 20302 sub_idx=20 pixel=[288.0,265.0]
+  <cand> 1 at token 20350 sub_idx=20 pixel=[279.0,208.0]
+  <cand> 2 at token 20398 sub_idx=20 pixel=[228.0,257.0]
+  <cand> 3 at token 20446 sub_idx=20 pixel=[218.0,120.0]
+  <cand> 4 at token 20494 sub_idx=20 pixel=[310.0,285.0]
+  <cand> 5 at token 20542 sub_idx=20 pixel=[264.0,292.0]
+  <cand> 0 at token 21056 sub_idx=21 pixel=[185.0,105.0]
+  <cand> 1 at token 21104 sub_idx=21 pixel=[153.0,152.0]
+  <cand> 2 at token 21152 sub_idx=21 pixel=[124.0,194.0]
+  <cand> 3 at token 21200 sub_idx=21 pixel=[130.0,188.0]
+  <cand> 4 at token 21248 sub_idx=21 pixel=[236.0,263.0]
+  <cand> 5 at token 21296 sub_idx=21 pixel=[168.0,195.0]
+  <cand> 0 at token 21810 sub_idx=22 pixel=[281.0,273.0]
+  <cand> 1 at token 21858 sub_idx=22 pixel=[164.0,266.0]
+  <cand> 2 at token 21906 sub_idx=22 pixel=[225.0,165.0]
+  <cand> 3 at token 21954 sub_idx=22 pixel=[199.0,132.0]
+  <cand> 4 at token 22002 sub_idx=22 pixel=[194.0,199.0]
+  <cand> 5 at token 22050 sub_idx=22 pixel=[139.0,173.0]
+  <cand> 6 at token 22098 sub_idx=22 pixel=[149.0,190.0]
+  <cand> 7 at token 22146 sub_idx=22 pixel=[262.0,291.0]
+  <cand> 8 at token 22194 sub_idx=22 pixel=[189.0,155.0]
+  <cand> 9 at token 22242 sub_idx=22 pixel=[294.0,295.0]
+  <cand> 0 at token 22757 sub_idx=23 pixel=[246.0,171.0]
+  <cand> 1 at token 22805 sub_idx=23 pixel=[247.0,101.0]
+  <cand> 2 at token 22853 sub_idx=23 pixel=[236.0,329.0]
+  <cand> 3 at token 22901 sub_idx=23 pixel=[288.0,203.0]
+  <cand> 4 at token 22949 sub_idx=23 pixel=[255.0,12.0]
+  <cand> 5 at token 22997 sub_idx=23 pixel=[282.0,81.0]
+  <cand> 6 at token 23045 sub_idx=23 pixel=[271.0,100.0]
+  <cand> 7 at token 23093 sub_idx=23 pixel=[205.0,246.0]
+  <cand> 0 at token 23605 sub_idx=24 pixel=[296.0,266.0]
+  <cand> 1 at token 23653 sub_idx=24 pixel=[283.0,192.0]
+  <cand> 2 at token 23701 sub_idx=24 pixel=[296.0,78.0]
+  <cand> 3 at token 23749 sub_idx=24 pixel=[344.0,184.0]
+  <cand> 4 at token 23797 sub_idx=24 pixel=[359.0,136.0]
+  <cand> 5 at token 23845 sub_idx=24 pixel=[242.0,146.0]
+  <cand> 6 at token 23893 sub_idx=24 pixel=[316.0,302.0]
+  <cand> 7 at token 23941 sub_idx=24 pixel=[270.0,103.0]
+  <cand> 0 at token 24455 sub_idx=25 pixel=[267.0,303.0]
+  <cand> 1 at token 24503 sub_idx=25 pixel=[199.0,248.0]
+  <e_cand> 0 at token 490 sub_idx=0 (same embed)
+  <e_cand> 1 at token 538 sub_idx=0 (same embed)
+  <e_cand> 2 at token 586 sub_idx=0 (same embed)
+  <e_cand> 0 at token 1198 sub_idx=1 (same embed)
+  <e_cand> 1 at token 1246 sub_idx=1 (same embed)
+  <e_cand> 2 at token 1294 sub_idx=1 (same embed)
+  <e_cand> 0 at token 2290 sub_idx=2 (same embed)
+  <e_cand> 1 at token 2338 sub_idx=2 (same embed)
+  <e_cand> 2 at token 2386 sub_idx=2 (same embed)
+  <e_cand> 0 at token 3091 sub_idx=3 (same embed)
+  <e_cand> 1 at token 3139 sub_idx=3 (same embed)
+  <e_cand> 2 at token 3187 sub_idx=3 (same embed)
+  <e_cand> 0 at token 3941 sub_idx=4 (same embed)
+  <e_cand> 1 at token 3989 sub_idx=4 (same embed)
+  <e_cand> 2 at token 4037 sub_idx=4 (same embed)
+  <e_cand> 0 at token 5427 sub_idx=5 (same embed)
+  <e_cand> 1 at token 5475 sub_idx=5 (same embed)
+  <e_cand> 2 at token 5523 sub_idx=5 (same embed)
+  <e_cand> 0 at token 6423 sub_idx=6 (same embed)
+  <e_cand> 1 at token 6471 sub_idx=6 (same embed)
+  <e_cand> 2 at token 6519 sub_idx=6 (same embed)
+  <e_cand> 0 at token 7462 sub_idx=7 (same embed)
+  <e_cand> 1 at token 7510 sub_idx=7 (same embed)
+  <e_cand> 2 at token 7558 sub_idx=7 (same embed)
+  <e_cand> 0 at token 8314 sub_idx=8 (same embed)
+  <e_cand> 1 at token 8362 sub_idx=8 (same embed)
+  <e_cand> 2 at token 8410 sub_idx=8 (same embed)
+  <e_cand> 0 at token 9307 sub_idx=9 (same embed)
+  <e_cand> 1 at token 9355 sub_idx=9 (same embed)
+  <e_cand> 2 at token 9403 sub_idx=9 (same embed)
+  <e_cand> 0 at token 10940 sub_idx=10 (same embed)
+  <e_cand> 1 at token 10988 sub_idx=10 (same embed)
+  <e_cand> 2 at token 11036 sub_idx=10 (same embed)
+  <e_cand> 0 at token 12179 sub_idx=11 (same embed)
+  <e_cand> 1 at token 12227 sub_idx=11 (same embed)
+  <e_cand> 2 at token 12275 sub_idx=11 (same embed)
+  <e_cand> 0 at token 13125 sub_idx=12 (same embed)
+  <e_cand> 1 at token 13173 sub_idx=12 (same embed)
+  <e_cand> 2 at token 13219 sub_idx=12 (same embed)
+  <e_cand> 0 at token 14020 sub_idx=13 (same embed)
+  <e_cand> 1 at token 14068 sub_idx=13 (same embed)
+  <e_cand> 2 at token 14116 sub_idx=13 (same embed)
+  <e_cand> 0 at token 14966 sub_idx=14 (same embed)
+  <e_cand> 1 at token 15014 sub_idx=14 (same embed)
+  <e_cand> 2 at token 15062 sub_idx=14 (same embed)
+  <e_cand> 0 at token 15864 sub_idx=15 (same embed)
+  <e_cand> 1 at token 15912 sub_idx=15 (same embed)
+  <e_cand> 2 at token 15960 sub_idx=15 (same embed)
+  <e_cand> 0 at token 16711 sub_idx=16 (same embed)
+  <e_cand> 1 at token 16759 sub_idx=16 (same embed)
+  <e_cand> 2 at token 16807 sub_idx=16 (same embed)
+  <e_cand> 0 at token 17512 sub_idx=17 (same embed)
+  <e_cand> 1 at token 17558 sub_idx=17 (same embed)
+  <e_cand> 2 at token 17606 sub_idx=17 (same embed)
+  <e_cand> 0 at token 18553 sub_idx=18 (same embed)
+  <e_cand> 1 at token 18601 sub_idx=18 (same embed)
+  <e_cand> 2 at token 18649 sub_idx=18 (same embed)
+  <e_cand> 0 at token 19545 sub_idx=19 (same embed)
+  <e_cand> 1 at token 19593 sub_idx=19 (same embed)
+  <e_cand> 2 at token 19641 sub_idx=19 (same embed)
+  <e_cand> 0 at token 20348 sub_idx=20 (same embed)
+  <e_cand> 1 at token 20396 sub_idx=20 (same embed)
+  <e_cand> 2 at token 20444 sub_idx=20 (same embed)
+  <e_cand> 0 at token 21102 sub_idx=21 (same embed)
+  <e_cand> 1 at token 21150 sub_idx=21 (same embed)
+  <e_cand> 2 at token 21198 sub_idx=21 (same embed)
+  <e_cand> 0 at token 21856 sub_idx=22 (same embed)
+  <e_cand> 1 at token 21904 sub_idx=22 (same embed)
+  <e_cand> 2 at token 21952 sub_idx=22 (same embed)
+  <e_cand> 0 at token 22803 sub_idx=23 (same embed)
+  <e_cand> 1 at token 22851 sub_idx=23 (same embed)
+  <e_cand> 2 at token 22899 sub_idx=23 (same embed)
+  <e_cand> 0 at token 23651 sub_idx=24 (same embed)
+  <e_cand> 1 at token 23699 sub_idx=24 (same embed)
+  <e_cand> 2 at token 23747 sub_idx=24 (same embed)
+  <e_cand> 0 at token 24501 sub_idx=25 (same embed)
+  <e_cand> 1 at token 24549 sub_idx=25 (same embed)
+
+[2026-02-25 06:18:21,881] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.45 | optimizer_gradients: 1.44 | optimizer_step: 0.87
+[2026-02-25 06:18:21,883] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 7101.76 | bwd_microstep: 10975.31 | bwd_inner_microstep: 8633.30 | bwd_allreduce_microstep: 2341.92 | step_microstep: 91.95
+[2026-02-25 06:18:21,884] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 7101.77 | bwd: 10975.31 | bwd_inner: 8633.29 | bwd_allreduce: 2341.94 | step: 91.96
+{'loss': 1.5964, 'learning_rate': 9.09090909090909e-07, 'epoch': 0.0}
+[2026-02-25 06:18:39,023] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.56 | optimizer_gradients: 2.00 | optimizer_step: 0.95
+[2026-02-25 06:18:39,025] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 8051.87 | bwd_microstep: 7784.85 | bwd_inner_microstep: 6309.51 | bwd_allreduce_microstep: 1475.23 | step_microstep: 593.61
+[2026-02-25 06:18:39,026] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 8051.89 | bwd: 7784.86 | bwd_inner: 6309.50 | bwd_allreduce: 1475.26 | step: 593.61
+{'loss': 1.659, 'learning_rate': 1.2121212121212122e-06, 'epoch': 0.0}
+[2026-02-25 06:18:52,805] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.19 | optimizer_gradients: 1.10 | optimizer_step: 0.81
+[2026-02-25 06:18:52,806] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6282.63 | bwd_microstep: 6950.30 | bwd_inner_microstep: 5828.60 | bwd_allreduce_microstep: 1121.60 | step_microstep: 18.40
+[2026-02-25 06:18:52,806] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6282.64 | bwd: 6950.29 | bwd_inner: 5828.59 | bwd_allreduce: 1121.62 | step: 18.40
+{'loss': 1.5848, 'learning_rate': 1.5151515151515152e-06, 'epoch': 0.0}
+[2026-02-25 06:19:05,866] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.29 | optimizer_gradients: 1.11 | optimizer_step: 0.81
+[2026-02-25 06:19:05,867] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6410.25 | bwd_microstep: 6271.65 | bwd_inner_microstep: 6195.07 | bwd_allreduce_microstep: 76.49 | step_microstep: 18.83
+[2026-02-25 06:19:05,867] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6410.27 | bwd: 6271.65 | bwd_inner: 6195.06 | bwd_allreduce: 76.51 | step: 18.84
+{'loss': 1.6153, 'learning_rate': 1.818181818181818e-06, 'epoch': 0.0}
+[2026-02-25 06:19:18,839] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.14 | optimizer_gradients: 1.05 | optimizer_step: 0.82
+[2026-02-25 06:19:18,840] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6342.45 | bwd_microstep: 6246.72 | bwd_inner_microstep: 6242.33 | bwd_allreduce_microstep: 4.30 | step_microstep: 18.15
+[2026-02-25 06:19:18,840] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6342.46 | bwd: 6246.72 | bwd_inner: 6242.32 | bwd_allreduce: 4.32 | step: 18.16
+{'loss': 1.6416, 'learning_rate': 2.1212121212121216e-06, 'epoch': 0.0}
+[2026-02-25 06:19:31,976] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.14 | optimizer_gradients: 1.07 | optimizer_step: 0.83
+[2026-02-25 06:19:31,978] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6311.47 | bwd_microstep: 6453.40 | bwd_inner_microstep: 5833.63 | bwd_allreduce_microstep: 619.70 | step_microstep: 18.15
+[2026-02-25 06:19:31,978] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6311.47 | bwd: 6453.41 | bwd_inner: 5833.62 | bwd_allreduce: 619.72 | step: 18.16
+{'loss': 1.5499, 'learning_rate': 2.4242424242424244e-06, 'epoch': 0.0}
+[2026-02-25 06:19:44,893] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.14 | optimizer_gradients: 1.15 | optimizer_step: 0.83
+[2026-02-25 06:19:44,894] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6273.18 | bwd_microstep: 6234.87 | bwd_inner_microstep: 6058.88 | bwd_allreduce_microstep: 175.92 | step_microstep: 21.21
+[2026-02-25 06:19:44,895] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6273.18 | bwd: 6234.87 | bwd_inner: 6058.87 | bwd_allreduce: 175.93 | step: 21.22
+{'loss': 1.4567, 'learning_rate': 2.7272727272727272e-06, 'epoch': 0.0}
+[2026-02-25 06:19:57,940] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.25 | optimizer_gradients: 1.12 | optimizer_step: 0.83
+[2026-02-25 06:19:57,941] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6458.85 | bwd_microstep: 6214.52 | bwd_inner_microstep: 6210.23 | bwd_allreduce_microstep: 4.20 | step_microstep: 19.43
+[2026-02-25 06:19:57,942] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6458.86 | bwd: 6214.52 | bwd_inner: 6210.23 | bwd_allreduce: 4.21 | step: 19.43
+{'loss': 1.3746, 'learning_rate': 3.0303030303030305e-06, 'epoch': 0.0}
+[2026-02-25 06:20:10,944] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.22 | optimizer_gradients: 1.08 | optimizer_step: 0.82
+[2026-02-25 06:20:10,946] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6353.43 | bwd_microstep: 6286.84 | bwd_inner_microstep: 6182.82 | bwd_allreduce_microstep: 103.94 | step_microstep: 19.10
+[2026-02-25 06:20:10,946] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6353.43 | bwd: 6286.84 | bwd_inner: 6182.81 | bwd_allreduce: 103.96 | step: 19.10
+{'loss': 1.2777, 'learning_rate': 3.3333333333333333e-06, 'epoch': 0.0}
+[2026-02-25 06:20:23,892] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.24 | optimizer_gradients: 1.05 | optimizer_step: 0.82
+[2026-02-25 06:20:23,893] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6333.80 | bwd_microstep: 6194.44 | bwd_inner_microstep: 6190.09 | bwd_allreduce_microstep: 4.27 | step_microstep: 19.16
+[2026-02-25 06:20:23,894] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6333.80 | bwd: 6194.45 | bwd_inner: 6190.08 | bwd_allreduce: 4.29 | step: 19.16
+{'loss': 1.3564, 'learning_rate': 3.636363636363636e-06, 'epoch': 0.0}
+[2026-02-25 06:20:36,837] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.35 | optimizer_gradients: 1.09 | optimizer_step: 0.81
+[2026-02-25 06:20:36,839] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6388.67 | bwd_microstep: 6187.08 | bwd_inner_microstep: 6157.39 | bwd_allreduce_microstep: 29.60 | step_microstep: 20.79
+[2026-02-25 06:20:36,839] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6388.68 | bwd: 6187.08 | bwd_inner: 6157.38 | bwd_allreduce: 29.62 | step: 20.79
+{'loss': 1.275, 'learning_rate': 3.939393939393939e-06, 'epoch': 0.0}
+[2026-02-25 06:20:49,744] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.37 | optimizer_gradients: 1.12 | optimizer_step: 0.75
+[2026-02-25 06:20:49,745] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6251.74 | bwd_microstep: 6165.91 | bwd_inner_microstep: 6117.25 | bwd_allreduce_microstep: 48.59 | step_microstep: 19.19
+[2026-02-25 06:20:49,745] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6251.75 | bwd: 6165.92 | bwd_inner: 6117.23 | bwd_allreduce: 48.61 | step: 19.19
+{'loss': 1.2457, 'learning_rate': 4.242424242424243e-06, 'epoch': 0.0}
+[2026-02-25 06:21:02,658] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.38 | optimizer_gradients: 1.08 | optimizer_step: 0.80
+[2026-02-25 06:21:02,659] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6326.26 | bwd_microstep: 6123.43 | bwd_inner_microstep: 6082.66 | bwd_allreduce_microstep: 40.68 | step_microstep: 19.10
+[2026-02-25 06:21:02,659] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6326.27 | bwd: 6123.42 | bwd_inner: 6082.65 | bwd_allreduce: 40.70 | step: 19.10
+{'loss': 1.2058, 'learning_rate': 4.5454545454545455e-06, 'epoch': 0.0}
+[2026-02-25 06:21:15,601] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.30 | optimizer_gradients: 1.12 | optimizer_step: 0.79
+[2026-02-25 06:21:15,602] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6394.92 | bwd_microstep: 6104.41 | bwd_inner_microstep: 6100.11 | bwd_allreduce_microstep: 4.21 | step_microstep: 18.76
+[2026-02-25 06:21:15,603] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6394.93 | bwd: 6104.40 | bwd_inner: 6100.10 | bwd_allreduce: 4.22 | step: 18.76
+{'loss': 1.1158, 'learning_rate': 4.848484848484849e-06, 'epoch': 0.0}
+[2026-02-25 06:21:28,722] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.22 | optimizer_gradients: 1.14 | optimizer_step: 0.84
+[2026-02-25 06:21:28,724] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6421.19 | bwd_microstep: 6232.89 | bwd_inner_microstep: 6228.60 | bwd_allreduce_microstep: 4.21 | step_microstep: 22.74
+[2026-02-25 06:21:28,724] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6421.21 | bwd: 6232.89 | bwd_inner: 6228.59 | bwd_allreduce: 4.22 | step: 22.82
+{'loss': 0.985, 'learning_rate': 5.151515151515152e-06, 'epoch': 0.0}
+[2026-02-25 06:21:41,721] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.23 | optimizer_gradients: 1.22 | optimizer_step: 0.87
+[2026-02-25 06:21:41,723] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6388.94 | bwd_microstep: 6156.86 | bwd_inner_microstep: 6152.26 | bwd_allreduce_microstep: 4.52 | step_microstep: 20.28
+[2026-02-25 06:21:41,723] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6388.94 | bwd: 6156.86 | bwd_inner: 6152.25 | bwd_allreduce: 4.54 | step: 20.28
+{'loss': 0.992, 'learning_rate': 5.4545454545454545e-06, 'epoch': 0.0}
+[2026-02-25 06:21:54,724] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.24 | optimizer_gradients: 1.14 | optimizer_step: 0.82
+[2026-02-25 06:21:54,725] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6390.08 | bwd_microstep: 6187.04 | bwd_inner_microstep: 6182.74 | bwd_allreduce_microstep: 4.21 | step_microstep: 18.93
+[2026-02-25 06:21:54,725] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6390.09 | bwd: 6187.04 | bwd_inner: 6182.73 | bwd_allreduce: 4.23 | step: 18.98
+{'loss': 0.8835, 'learning_rate': 5.7575757575757586e-06, 'epoch': 0.0}
+[2026-02-25 06:22:07,716] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.27 | optimizer_gradients: 1.15 | optimizer_step: 0.81
+[2026-02-25 06:22:07,718] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6356.84 | bwd_microstep: 6207.92 | bwd_inner_microstep: 6092.72 | bwd_allreduce_microstep: 115.11 | step_microstep: 18.93
+[2026-02-25 06:22:07,718] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6356.85 | bwd: 6207.92 | bwd_inner: 6092.70 | bwd_allreduce: 115.13 | step: 18.93
+{'loss': 0.992, 'learning_rate': 6.060606060606061e-06, 'epoch': 0.0}
+[2026-02-25 06:22:20,554] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | optimizer_allgather: 1.24 | optimizer_gradients: 1.15 | optimizer_step: 0.82
+[2026-02-25 06:22:20,556] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd_microstep: 6331.32 | bwd_microstep: 6050.91 | bwd_inner_microstep: 6011.84 | bwd_allreduce_microstep: 38.99 | step_microstep: 20.64
+[2026-02-25 06:22:20,556] [INFO] [logging.py:96:log_dist] [Rank 0] time (ms) | fwd: 6331.33 | bwd: 6050.91 | bwd_inner: 6011.83 | bwd_allreduce: 39.00 | step: 20.64
+{'loss': 0.939, 'learning_rate': 6.363636363636363e-06, 'epoch': 0.0}
+  File "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py", line 764, in <module>
+    main()
+  File "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py", line 749, in main
+    train_result = trainer.train(resume_from_checkpoint=checkpoint)
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/transformers/trainer.py", line 1539, in train
+    return inner_training_loop(
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/transformers/trainer.py", line 1836, in _inner_training_loop
+    for step, inputs in enumerate(epoch_iterator):
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 701, in __next__
+    data = self._next_data()
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1445, in _next_data
+    return self._process_data(data)
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1491, in _process_data
+    data.reraise()
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/_utils.py", line 715, in reraise
+    raise exception
+RuntimeError: Caught RuntimeError in pin memory thread for device 0.
+Original Traceback (most recent call last):
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 41, in do_one_step
+    data = pin_memory(data, device)
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 75, in pin_memory
+    {k: pin_memory(sample, device) for k, sample in data.items()}
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 75, in <dictcomp>
+    {k: pin_memory(sample, device) for k, sample in data.items()}
+  File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 64, in pin_memory
+    return data.pin_memory(device)
+RuntimeError: CUDA error: out of memory
+CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.
+
+
+[rank0]: Traceback (most recent call last):
+[rank0]:   File "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py", line 764, in <module>
+[rank0]:     main()
+[rank0]:   File "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py", line 749, in main
+[rank0]:     train_result = trainer.train(resume_from_checkpoint=checkpoint)
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/transformers/trainer.py", line 1539, in train
+[rank0]:     return inner_training_loop(
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/transformers/trainer.py", line 1836, in _inner_training_loop
+[rank0]:     for step, inputs in enumerate(epoch_iterator):
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 701, in __next__
+[rank0]:     data = self._next_data()
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1445, in _next_data
+[rank0]:     return self._process_data(data)
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/dataloader.py", line 1491, in _process_data
+[rank0]:     data.reraise()
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/_utils.py", line 715, in reraise
+[rank0]:     raise exception
+[rank0]: RuntimeError: Caught RuntimeError in pin memory thread for device 0.
+[rank0]: Original Traceback (most recent call last):
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 41, in do_one_step
+[rank0]:     data = pin_memory(data, device)
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 75, in pin_memory
+[rank0]:     {k: pin_memory(sample, device) for k, sample in data.items()}
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 75, in <dictcomp>
+[rank0]:     {k: pin_memory(sample, device) for k, sample in data.items()}
+[rank0]:   File "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/lib/python3.10/site-packages/torch/utils/data/_utils/pin_memory.py", line 64, in pin_memory
+[rank0]:     return data.pin_memory(device)
+[rank0]: RuntimeError: CUDA error: out of memory
+[rank0]: CUDA kernel errors might be asynchronously reported at some other API call, so the stacktrace below might be incorrect.
+[rank0]: For debugging consider passing CUDA_LAUNCH_BLOCKING=1
+[rank0]: Compile with `TORCH_USE_CUDA_DSA` to enable device-side assertions.

wandb/run-20260225_055938-u2vegsv1/files/requirements.txt

@@ -0,0 +1,113 @@
+nvidia-cuda-runtime-cu12==12.1.105
+platformdirs==4.9.2
+wandb==0.22.3
+transformers==4.37.2
+einops==0.8.2
+accelerate==0.33.0
+aiosignal==1.4.0
+huggingface_hub==0.36.2
+pycparser==2.22
+multidict==6.7.1
+urllib3==2.6.3
+urllib3==2.5.0
+aiohappyeyeballs==2.6.1
+nvidia-cusolver-cu12==11.4.5.107
+yarl==1.22.0
+typing-inspection==0.4.2
+gitdb==4.0.12
+multiprocess==0.70.16
+async-timeout==5.0.1
+Jinja2==3.1.6
+setuptools==82.0.0
+torch==2.5.1+cu121
+hyperframe==6.1.0
+peft==0.10.0
+tzdata==2025.3
+dill==0.3.8
+orjson==3.11.7
+nvidia-cufft-cu12==11.0.2.54
+propcache==0.4.1
+PySocks==1.7.1
+nvidia-cublas-cu12==12.1.3.1
+nvidia-nccl-cu12==2.21.5
+docker-pycreds==0.4.0
+pydantic_core==2.41.5
+nvidia-ml-py==13.590.48
+ninja==1.13.0
+cffi==1.15.0
+nvidia-cuda-cupti-cu12==12.1.105
+triton==3.1.0
+annotated-types==0.7.0
+nvidia-nvtx-cu12==12.1.105
+timm==1.0.22
+sympy==1.13.1
+pydantic==2.12.5
+xxhash==3.6.0
+py-cpuinfo==9.0.0
+sentry-sdk==2.53.0
+networkx==3.4.2
+click==8.3.1
+regex==2026.1.15
+pillow==12.0.0
+zstandard==0.23.0
+pyarrow==20.0.0
+GitPython==3.1.46
+wheel==0.46.3
+fsspec==2025.3.0
+typing_extensions==4.15.0
+hf-xet==1.2.0
+torchvision==0.20.1+cu121
+python-dateutil==2.9.0.post0
+numpy==1.26.4
+smmap==5.0.2
+setproctitle==1.2.3
+safetensors==0.7.0
+requests==2.32.5
+hpack==4.1.0
+six==1.17.0
+modelscope==1.34.0
+pip==26.0.1
+h2==4.3.0
+PyYAML==6.0.3
+PyYAML==6.0
+torchaudio==2.5.1+cu121
+nvidia-nvjitlink-cu12==12.9.86
+psutil==7.2.2
+psutil==5.9.1
+pandas==2.3.2
+pytz==2025.2
+nvidia-cudnn-cu12==9.1.0.70
+packaging==25.0
+hjson==3.1.0
+deepspeed==0.14.4
+datasets==4.0.0
+nvidia-cuda-nvrtc-cu12==12.1.105
+tokenizers==0.15.2
+MarkupSafe==2.1.5
+charset-normalizer==3.4.4
+frozenlist==1.8.0
+Brotli==1.0.9
+certifi==2026.1.4
+mpmath==1.3.0
+protobuf==3.20.1
+tqdm==4.67.3
+nvidia-cusparse-cu12==12.1.0.106
+attrs==25.4.0
+appdirs==1.4.4
+flash_attn==2.7.4.post1
+aiohttp==3.13.3
+filelock==3.20.0
+idna==3.11
+nvidia-curand-cu12==10.3.2.106
+zipp==3.23.0
+backports.tarfile==1.2.0
+autocommand==2.2.2
+platformdirs==4.4.0
+jaraco.text==4.0.0
+more-itertools==10.8.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+wheel==0.46.3
+tomli==2.4.0
+packaging==26.0
+importlib_metadata==8.7.1

wandb/run-20260225_055938-u2vegsv1/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-3.10.0-957.el7.x86_64-x86_64-with-glibc2.17",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2026-02-24T21:59:38.884470Z",
+  "args":  [
+    "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/resolved_train_config.json"
+  ],
+  "program":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py",
+  "codePath":  "InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py",
+  "codePathLocal":  "internvl_cleaned/train/internvl_chat_finetune.py",
+  "git":  {
+    "remote":  "git@github.com:Yuxin916/CL_CoTNav.git",
+    "commit":  "f7d6fbe6a8031d29a95c2f3be79e99f96670a12e"
+  },
+  "email":  "caiy0039@e.ntu.edu.sg",
+  "root":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY",
+  "host":  "SH-IDC1-10-140-37-45",
+  "executable":  "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/bin/python3.10",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "524945911808",
+      "used":  "39460401152"
+    }
+  },
+  "memory":  {
+    "total":  "1081627828224"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-9e78ad4b-b304-a199-38f4-24b3acd9b531"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-19c9429b-5ecf-0b76-7d81-03c36a449f32"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-57520d78-d04a-3028-bd40-5ebd78e123e6"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-0d94270a-8ad3-b3a6-4acd-145fccc36d85"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-b9b6fe80-e37a-b622-0deb-27ef46a965ff"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-7b870038-dbe6-4039-b6bf-f90517f43f6c"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-e3065d01-9539-64c9-417f-91273b521051"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-28ce6eed-85b8-155a-5a5d-412dfb1dd1c0"
+    }
+  ],
+  "cudaVersion":  "12.2",
+  "slurm":  {
+    "cluster_name":  "cluster_sproject3",
+    "conf":  "/etc/slurm/slurm.conf",
+    "cpus_on_node":  "128",
+    "cpus_per_task":  "8",
+    "distribution":  "cyclic",
+    "gtids":  "0",
+    "job_account":  "research",
+    "job_cpus_per_node":  "128",
+    "job_cpus_per_node_pack_group_0":  "128",
+    "job_gid":  "200000139",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "7464467",
+    "job_name":  "vlm_ft",
+    "job_nodelist":  "SH-IDC1-10-140-37-45",
+    "job_num_nodes":  "1",
+    "job_partition":  "interntmp",
+    "job_qos":  "normal",
+    "job_uid":  "200000139",
+    "job_user":  "wangmaonan",
+    "jobid":  "7464467",
+    "launch_node_ipaddr":  "10.140.37.45",
+    "localid":  "0",
+    "mem_per_node":  "49152",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "SH-IDC1-10-140-37-45",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "srun_comm_host":  "10.140.37.45",
+    "srun_comm_port":  "44377",
+    "step_gpus":  "0,1,2,3",
+    "step_id":  "0",
+    "step_launcher_port":  "44377",
+    "step_nodelist":  "SH-IDC1-10-140-37-45",
+    "step_num_nodes":  "1",
+    "step_num_tasks":  "1",
+    "step_tasks_per_node":  "1",
+    "stepid":  "0",
+    "submit_dir":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav",
+    "submit_host":  "SH-IDC1-10-140-37-45",
+    "task_pid":  "151799",
+    "tasks_per_node":  "1",
+    "topology_addr":  "SH-IDC1-10-140-37-45",
+    "topology_addr_pattern":  "node",
+    "umask":  "0002",
+    "working_cluster":  "cluster_sproject3:SH-IDC1-10-140-37-161:6817:9216:109"
+  },
+  "writerId":  "lcgegjlerikh15wodksbpfheadlvwekl"
+}

wandb/run-20260225_055938-u2vegsv1/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/epoch":0,"train/global_step":21,"_wandb":{"runtime":1360},"_runtime":1360,"_timestamp":1.7719717405855756e+09,"_step":20,"train/loss":0.939,"train/learning_rate":6.363636363636363e-06}

wandb/run-20260225_055938-u2vegsv1/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2026-02-25T05:59:39.04618224+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp546zrv3n/port-154283.txt","pid":154283,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-02-25T05:59:39.04849989+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":154283}
+{"time":"2026-02-25T05:59:39.048638993+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-154283-157255-155222642/socket","Net":"unix"}}
+{"time":"2026-02-25T05:59:39.129695518+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-02-25T05:59:39.140549387+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"u2vegsv1","id":"1(@)"}
+{"time":"2026-02-25T05:59:39.556207953+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"u2vegsv1","id":"1(@)"}
+{"time":"2026-02-25T06:22:20.609960124+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-02-25T06:22:20.611818219+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-02-25T06:22:20.611755671+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-02-25T06:22:20.612915442+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-154283-157255-155222642/socket","Net":"unix"}}
+{"time":"2026-02-25T06:22:20.613364891+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-02-25T06:22:22.357517871+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-02-25T06:22:22.35822517+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-02-25T06:22:22.358768645+08:00","level":"INFO","msg":"server is closed"}

wandb/run-20260225_055938-u2vegsv1/logs/debug-internal.log

@@ -0,0 +1,11 @@
+{"time":"2026-02-25T05:59:39.14150883+08:00","level":"INFO","msg":"stream: starting","core version":"0.22.3"}
+{"time":"2026-02-25T05:59:39.552925091+08:00","level":"INFO","msg":"stream: created new stream","id":"u2vegsv1"}
+{"time":"2026-02-25T05:59:39.553478946+08:00","level":"INFO","msg":"handler: started","stream_id":"u2vegsv1"}
+{"time":"2026-02-25T05:59:39.555784131+08:00","level":"INFO","msg":"stream: started","id":"u2vegsv1"}
+{"time":"2026-02-25T05:59:39.555788317+08:00","level":"INFO","msg":"writer: started","stream_id":"u2vegsv1"}
+{"time":"2026-02-25T05:59:39.555793631+08:00","level":"INFO","msg":"sender: started","stream_id":"u2vegsv1"}
+{"time":"2026-02-25T06:22:20.612437616+08:00","level":"INFO","msg":"stream: closing","id":"u2vegsv1"}
+{"time":"2026-02-25T06:22:21.743357445+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-02-25T06:22:22.353551338+08:00","level":"INFO","msg":"handler: closed","stream_id":"u2vegsv1"}
+{"time":"2026-02-25T06:22:22.355293739+08:00","level":"INFO","msg":"sender: closed","stream_id":"u2vegsv1"}
+{"time":"2026-02-25T06:22:22.355734583+08:00","level":"INFO","msg":"stream: closed","id":"u2vegsv1"}

wandb/run-20260225_055938-u2vegsv1/logs/debug.log

@@ -0,0 +1,24 @@
+2026-02-25 05:59:38,900 INFO    MainThread:154283 [wandb_setup.py:_flush():81] Current SDK version is 0.22.3
+2026-02-25 05:59:38,902 INFO    MainThread:154283 [wandb_setup.py:_flush():81] Configure stats pid to 154283
+2026-02-25 05:59:38,903 INFO    MainThread:154283 [wandb_setup.py:_flush():81] Loading settings from /mnt/petrelfs/wangmaonan/.config/wandb/settings
+2026-02-25 05:59:38,903 INFO    MainThread:154283 [wandb_setup.py:_flush():81] Loading settings from /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/wandb/settings
+2026-02-25 05:59:38,904 INFO    MainThread:154283 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-02-25 05:59:38,904 INFO    MainThread:154283 [wandb_init.py:setup_run_log_directory():706] Logging user logs to /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/wandb/run-20260225_055938-u2vegsv1/logs/debug.log
+2026-02-25 05:59:38,905 INFO    MainThread:154283 [wandb_init.py:setup_run_log_directory():707] Logging internal logs to /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/wandb/run-20260225_055938-u2vegsv1/logs/debug-internal.log
+2026-02-25 05:59:38,905 INFO    MainThread:154283 [wandb_init.py:init():833] calling init triggers
+2026-02-25 05:59:38,906 INFO    MainThread:154283 [wandb_init.py:init():838] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-02-25 05:59:38,906 INFO    MainThread:154283 [wandb_init.py:init():881] starting backend
+2026-02-25 05:59:39,129 INFO    MainThread:154283 [wandb_init.py:init():884] sending inform_init request
+2026-02-25 05:59:39,136 INFO    MainThread:154283 [wandb_init.py:init():892] backend started and connected
+2026-02-25 05:59:39,138 INFO    MainThread:154283 [wandb_init.py:init():962] updated telemetry
+2026-02-25 05:59:39,163 INFO    MainThread:154283 [wandb_init.py:init():986] communicating run to backend with 90.0 second timeout
+2026-02-25 05:59:40,117 INFO    MainThread:154283 [wandb_init.py:init():1033] starting run threads in backend
+2026-02-25 05:59:40,400 INFO    MainThread:154283 [wandb_run.py:_console_start():2506] atexit reg
+2026-02-25 05:59:40,400 INFO    MainThread:154283 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2026-02-25 05:59:40,401 INFO    MainThread:154283 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2026-02-25 05:59:40,401 INFO    MainThread:154283 [wandb_run.py:_redirect():2446] Redirects installed.
+2026-02-25 05:59:40,407 INFO    MainThread:154283 [wandb_init.py:init():1073] run started, returning control to user process
+2026-02-25 05:59:40,410 INFO    MainThread:154283 [wandb_run.py:_config_callback():1390] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'torch.bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['InternVLChatModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '../pretrained/InternVL3-2B', '_commit_hash': None, '_attn_implementation_internal': None, 'transformers_version': None, 'auto_map': {'AutoConfig': 'configuration_internvl_chat.InternVLChatConfig', 'AutoModel': 'modeling_internvl_chat.InternVLChatModel', 'AutoModelForCausalLM': 'modeling_internvl_chat.InternVLChatModel'}, 'hidden_size': 1536, 'image_fold': None, 'model_type': 'internvl_chat', 'system_message': 'You are an autonomous navigation agent operating in indoor environments. You receive spatial information through position embeddings injected into visual features and text tokens. Use the BEV map, position embeddings, and semantic information to make navigation decisions. When the target object is detected (<target> marker), navigate directly to it. Otherwise, explore frontiers strategically to find the goal object.', 'vision_config': {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': True, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['InternVisionModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'OpenGVLab/InternViT-6B-448px-V1-5', 'transformers_version': '4.37.2', '_attn_implementation_autoset': True, 'auto_map': {'AutoConfig': 'configuration_intern_vit.InternVisionConfig', 'AutoModel': 'modeling_intern_vit.InternVisionModel'}, 'capacity_factor': 1.2, 'eval_capacity_factor': 1.4, 'laux_allreduce': 'all_nodes', 'model_type': 'intern_vit_6b', 'moe_coeff_ratio': 0.5, 'moe_intermediate_size': 768, 'moe_output_scale': 4.0, 'noisy_gate_policy': 'RSample_before', 'num_experts': 8, 'num_routed_experts': 4, 'num_shared_experts': 4, 'shared_expert_intermediate_size': 3072, 'use_moe': False, 'use_residual': True, 'use_rts': False, 'use_weighted_residual': False, 'hidden_size': 1024, 'intermediate_size': 4096, 'dropout': 0.0, 'drop_path_rate': 0.0, 'num_hidden_layers': 24, 'num_attention_heads': 16, 'num_channels': 3, 'patch_size': 14, 'image_size': 448, 'initializer_range': 1e-10, 'initializer_factor': 0.1, 'attention_dropout': 0.0, 'layer_norm_eps': 1e-06, 'hidden_act': 'gelu', 'norm_type': 'layer_norm', 'qkv_bias': True, 'qk_normalization': False, 'use_flash_attn': True}, 'llm_config': {'vocab_size': 151677, 'max_position_embeddings': 32768, 'hidden_size': 1536, 'intermediate_size': 8960, 'num_hidden_layers': 28, 'num_attention_heads': 12, 'use_sliding_window': False, 'sliding_window': None, 'max_window_layers': 70, 'num_key_value_heads': 2, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 1000000.0, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': True, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['Qwen2ForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 151643, 'pad_token_id': None, 'eos_token_id': 151643, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': './pretrained/Qwen2.5-32B-Instruct', 'transformers_version': '4.37.2', '_attn_implementation_autoset': True, 'model_type': 'qwen2', 'moe_config': None, 'rope_scaling': {'factor': 2.0, 'rope_type': 'dynamic', 'type': 'dynamic'}, 'attn_implementation': 'flash_attention_2'}, 'use_backbone_lora': 0, 'use_llm_lora': 64, 'pad2square': False, 'select_layer': -1, 'force_image_size': 448, 'downsample_ratio': 0.5, 'template': 'internvl2_5_nav', 'dynamic_image_size': False, 'use_thumbnail': True, 'ps_version': 'v2', 'min_dynamic_patch': 1, 'max_dynamic_patch': 12, 'num_image_token_bev': 256, 'num_image_token_ego': 32, 'use_pairwise_spatial_encoder': False, 'use_position_embeddings': True, 'dual_text_pos_injection': True, 'bev_image_size': 448, 'vit_bev_freeze': True, 'vit_bev_use_lora': True, 'vit_bev_lora_rank': 64, 'vit_rgb_freeze': True, 'vit_rgb_use_lora': True, 'vit_rgb_lora_rank': 16, 'output_dir': '/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': 11000, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'runs/Feb25_05-58-51_SH-IDC1-10-140-37-45', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 0.5, 'save_total_limit': 2, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 12, 'past_index': -1, 'run_name': 'a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY_steps11000_gpus4_acc1', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'deepspeed': 'zero_stage2_config_acc1.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': False, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None}
+2026-02-25 06:22:20,606 INFO    wandb-AsyncioManager-main:154283 [service_client.py:_forward_responses():80] Reached EOF.
+2026-02-25 06:22:20,606 INFO    wandb-AsyncioManager-main:154283 [mailbox.py:close():137] Closing mailbox, abandoning 1 handles.

wandb/run-20260225_055938-u2vegsv1/run-u2vegsv1.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:465e04b8972492c2e154f9a5c50c1934ae6a535cad3a90ec8b25e3fa67d05e5c
+size 435093

wandb/run-20260225_063717-0ub00jhc/files/output.log

@@ -0,0 +1,398 @@
+  0%|          | 0/9300 [00:00<?, ?it/s]
+02/25/2026 06:53:48 - INFO - internvl_cleaned.model.internvl_chat.modeling_internvl_chat - [Text Position Embedding Verification] Sample 1:
+  <s> at token 437 sub_idx=0
+  <s> at token 1921 sub_idx=1
+  <s> at token 3160 sub_idx=2
+  <s> at token 4545 sub_idx=3
+  <s> at token 5493 sub_idx=4
+  <s> at token 6587 sub_idx=5
+  <s> at token 7484 sub_idx=6
+  <s> at token 8141 sub_idx=7
+  <s> at token 8945 sub_idx=8
+  <s> at token 9891 sub_idx=9
+  <s> at token 10835 sub_idx=10
+  <s> at token 12123 sub_idx=11
+  <s> at token 13020 sub_idx=12
+  <s> at token 13821 sub_idx=13
+  <s> at token 14575 sub_idx=14
+  <s> at token 15767 sub_idx=15
+  <s> at token 16617 sub_idx=16
+  <s> at token 17905 sub_idx=17
+  <s> at token 19243 sub_idx=18
+  <s> at token 20482 sub_idx=19
+  <s> at token 21331 sub_idx=20
+  <s> at token 22180 sub_idx=21
+  <s> at token 23030 sub_idx=22
+  <s> at token 23973 sub_idx=23
+  <e_s> at token 439 sub_idx=0
+  <e_s> at token 1923 sub_idx=1
+  <e_s> at token 3162 sub_idx=2
+  <e_s> at token 4547 sub_idx=3
+  <e_s> at token 5495 sub_idx=4
+  <e_s> at token 6589 sub_idx=5
+  <e_s> at token 7486 sub_idx=6
+  <e_s> at token 8143 sub_idx=7
+  <e_s> at token 8947 sub_idx=8
+  <e_s> at token 9893 sub_idx=9
+  <e_s> at token 10837 sub_idx=10
+  <e_s> at token 12125 sub_idx=11
+  <e_s> at token 13022 sub_idx=12
+  <e_s> at token 13823 sub_idx=13
+  <e_s> at token 14577 sub_idx=14
+  <e_s> at token 15769 sub_idx=15
+  <e_s> at token 16619 sub_idx=16
+  <e_s> at token 17907 sub_idx=17
+  <e_s> at token 19245 sub_idx=18
+  <e_s> at token 20484 sub_idx=19
+  <e_s> at token 21333 sub_idx=20
+  <e_s> at token 22182 sub_idx=21
+  <e_s> at token 23032 sub_idx=22
+  <e_s> at token 23975 sub_idx=23
+  <cand> 0 at token 444 sub_idx=0 pixel=[320.0,270.0]
+  <cand> 1 at token 492 sub_idx=0 pixel=[185.0,213.0]
+  <cand> 2 at token 540 sub_idx=0 pixel=[319.0,247.0]
+  <cand> 3 at token 588 sub_idx=0 pixel=[256.0,210.0]
+  <cand> 4 at token 636 sub_idx=0 pixel=[212.0,138.0]
+  <cand> 5 at token 684 sub_idx=0 pixel=[245.0,121.0]
+  <cand> 6 at token 732 sub_idx=0 pixel=[272.0,370.0]
+  <cand> 7 at token 780 sub_idx=0 pixel=[307.0,160.0]
+  <cand> 8 at token 828 sub_idx=0 pixel=[242.0,314.0]
+  <cand> 9 at token 876 sub_idx=0 pixel=[181.0,243.0]
+  <cand> 10 at token 924 sub_idx=0 pixel=[235.0,183.0]
+  <cand> 11 at token 973 sub_idx=0 pixel=[275.0,174.0]
+  <cand> 12 at token 1022 sub_idx=0 pixel=[240.0,152.0]
+  <cand> 13 at token 1071 sub_idx=0 pixel=[216.0,214.0]
+  <cand> 14 at token 1118 sub_idx=0 pixel=[315.0,117.0]
+  <cand> 15 at token 1167 sub_idx=0 pixel=[226.0,111.0]
+  <cand> 16 at token 1216 sub_idx=0 pixel=[189.0,277.0]
+  <cand> 17 at token 1265 sub_idx=0 pixel=[278.0,161.0]
+  <cand> 18 at token 1314 sub_idx=0 pixel=[203.0,286.0]
+  <cand> 19 at token 1363 sub_idx=0 pixel=[275.0,194.0]
+  <cand> 20 at token 1412 sub_idx=0 pixel=[404.0,400.0]
+  <cand> 0 at token 1928 sub_idx=1 pixel=[141.0,126.0]
+  <cand> 1 at token 1976 sub_idx=1 pixel=[109.0,55.0]
+  <cand> 2 at token 2024 sub_idx=1 pixel=[143.0,246.0]
+  <cand> 3 at token 2072 sub_idx=1 pixel=[224.0,269.0]
+  <cand> 4 at token 2120 sub_idx=1 pixel=[145.0,219.0]
+  <cand> 5 at token 2168 sub_idx=1 pixel=[175.0,188.0]
+  <cand> 6 at token 2216 sub_idx=1 pixel=[45.0,88.0]
+  <cand> 7 at token 2264 sub_idx=1 pixel=[57.0,178.0]
+  <cand> 8 at token 2312 sub_idx=1 pixel=[18.0,16.0]
+  <cand> 9 at token 2360 sub_idx=1 pixel=[97.0,76.0]
+  <cand> 10 at token 2408 sub_idx=1 pixel=[24.0,147.0]
+  <cand> 11 at token 2457 sub_idx=1 pixel=[161.0,107.0]
+  <cand> 12 at token 2506 sub_idx=1 pixel=[233.0,297.0]
+  <cand> 13 at token 2553 sub_idx=1 pixel=[100.0,222.0]
+  <cand> 14 at token 2602 sub_idx=1 pixel=[31.0,226.0]
+  <cand> 15 at token 2651 sub_idx=1 pixel=[224.0,255.0]
+  <cand> 0 at token 3167 sub_idx=2 pixel=[155.0,57.0]
+  <cand> 1 at token 3215 sub_idx=2 pixel=[6.0,64.0]
+  <cand> 2 at token 3263 sub_idx=2 pixel=[172.0,64.0]
+  <cand> 3 at token 3311 sub_idx=2 pixel=[174.0,148.0]
+  <cand> 4 at token 3359 sub_idx=2 pixel=[20.0,108.0]
+  <cand> 5 at token 3407 sub_idx=2 pixel=[86.0,136.0]
+  <cand> 6 at token 3455 sub_idx=2 pixel=[84.0,33.0]
+  <cand> 7 at token 3503 sub_idx=2 pixel=[91.0,69.0]
+  <cand> 8 at token 3551 sub_idx=2 pixel=[219.0,232.0]
+  <cand> 9 at token 3597 sub_idx=2 pixel=[28.0,85.0]
+  <cand> 10 at token 3645 sub_idx=2 pixel=[195.0,236.0]
+  <cand> 11 at token 3694 sub_idx=2 pixel=[29.0,44.0]
+  <cand> 12 at token 3743 sub_idx=2 pixel=[157.0,76.0]
+  <cand> 13 at token 3792 sub_idx=2 pixel=[38.0,129.0]
+  <cand> 14 at token 3841 sub_idx=2 pixel=[86.0,208.0]
+  <cand> 15 at token 3890 sub_idx=2 pixel=[168.0,211.0]
+  <cand> 16 at token 3939 sub_idx=2 pixel=[49.0,206.0]
+  <cand> 17 at token 3988 sub_idx=2 pixel=[149.0,230.0]
+  <cand> 18 at token 4037 sub_idx=2 pixel=[58.0,102.0]
+  <cand> 0 at token 4552 sub_idx=3 pixel=[21.0,189.0]
+  <cand> 1 at token 4600 sub_idx=3 pixel=[220.0,170.0]
+  <cand> 2 at token 4648 sub_idx=3 pixel=[5.0,225.0]
+  <cand> 3 at token 4696 sub_idx=3 pixel=[193.0,118.0]
+  <cand> 4 at token 4744 sub_idx=3 pixel=[180.0,151.0]
+  <cand> 5 at token 4792 sub_idx=3 pixel=[72.0,83.0]
+  <cand> 6 at token 4840 sub_idx=3 pixel=[157.0,221.0]
+  <cand> 7 at token 4888 sub_idx=3 pixel=[159.0,82.0]
+  <cand> 8 at token 4936 sub_idx=3 pixel=[99.0,179.0]
+  <cand> 9 at token 4984 sub_idx=3 pixel=[213.0,234.0]
+  <cand> 0 at token 5500 sub_idx=4 pixel=[278.0,132.0]
+  <cand> 1 at token 5548 sub_idx=4 pixel=[272.0,71.0]
+  <cand> 2 at token 5596 sub_idx=4 pixel=[278.0,266.0]
+  <cand> 3 at token 5644 sub_idx=4 pixel=[222.0,135.0]
+  <cand> 4 at token 5692 sub_idx=4 pixel=[208.0,113.0]
+  <cand> 5 at token 5740 sub_idx=4 pixel=[260.0,237.0]
+  <cand> 6 at token 5788 sub_idx=4 pixel=[175.0,258.0]
+  <cand> 7 at token 5836 sub_idx=4 pixel=[229.0,282.0]
+  <cand> 8 at token 5884 sub_idx=4 pixel=[291.0,200.0]
+  <cand> 9 at token 5932 sub_idx=4 pixel=[235.0,65.0]
+  <cand> 10 at token 5980 sub_idx=4 pixel=[269.0,287.0]
+  <cand> 11 at token 6029 sub_idx=4 pixel=[207.0,86.0]
+  <cand> 12 at token 6078 sub_idx=4 pixel=[183.0,242.0]
+  <cand> 0 at token 6594 sub_idx=5 pixel=[312.0,203.0]
+  <cand> 1 at token 6642 sub_idx=5 pixel=[203.0,237.0]
+  <cand> 2 at token 6690 sub_idx=5 pixel=[230.0,143.0]
+  <cand> 3 at token 6738 sub_idx=5 pixel=[342.0,209.0]
+  <cand> 4 at token 6786 sub_idx=5 pixel=[247.0,178.0]
+  <cand> 5 at token 6834 sub_idx=5 pixel=[216.0,115.0]
+  <cand> 6 at token 6882 sub_idx=5 pixel=[308.0,249.0]
+  <cand> 7 at token 6930 sub_idx=5 pixel=[263.0,258.0]
+  <cand> 8 at token 6978 sub_idx=5 pixel=[218.0,197.0]
+  <cand> 0 at token 7491 sub_idx=6 pixel=[173.0,163.0]
+  <cand> 1 at token 7539 sub_idx=6 pixel=[253.0,129.0]
+  <cand> 2 at token 7587 sub_idx=6 pixel=[192.0,203.0]
+  <cand> 3 at token 7635 sub_idx=6 pixel=[193.0,115.0]
+  <cand> 0 at token 8148 sub_idx=7 pixel=[269.0,204.0]
+  <cand> 1 at token 8196 sub_idx=7 pixel=[161.0,221.0]
+  <cand> 2 at token 8244 sub_idx=7 pixel=[193.0,296.0]
+  <cand> 3 at token 8292 sub_idx=7 pixel=[159.0,251.0]
+  <cand> 4 at token 8340 sub_idx=7 pixel=[139.0,222.0]
+  <cand> 5 at token 8388 sub_idx=7 pixel=[271.0,231.0]
+  <cand> 6 at token 8436 sub_idx=7 pixel=[242.0,235.0]
+  <cand> 0 at token 8952 sub_idx=8 pixel=[234.0,193.0]
+  <cand> 1 at token 9000 sub_idx=8 pixel=[211.0,399.0]
+  <cand> 2 at token 9048 sub_idx=8 pixel=[204.0,379.0]
+  <cand> 3 at token 9096 sub_idx=8 pixel=[217.0,126.0]
+  <cand> 4 at token 9144 sub_idx=8 pixel=[197.0,206.0]
+  <cand> 5 at token 9192 sub_idx=8 pixel=[272.0,344.0]
+  <cand> 6 at token 9240 sub_idx=8 pixel=[241.0,177.0]
+  <cand> 7 at token 9288 sub_idx=8 pixel=[303.0,416.0]
+  <cand> 8 at token 9336 sub_idx=8 pixel=[194.0,316.0]
+  <cand> 9 at token 9384 sub_idx=8 pixel=[269.0,443.0]
+  <cand> 0 at token 9898 sub_idx=9 pixel=[248.0,271.0]
+  <cand> 1 at token 9946 sub_idx=9 pixel=[183.0,251.0]
+  <cand> 2 at token 9994 sub_idx=9 pixel=[243.0,244.0]
+  <cand> 3 at token 10042 sub_idx=9 pixel=[251.0,312.0]
+  <cand> 4 at token 10090 sub_idx=9 pixel=[230.0,214.0]
+  <cand> 5 at token 10136 sub_idx=9 pixel=[107.0,373.0]
+  <cand> 6 at token 10184 sub_idx=9 pixel=[110.0,350.0]
+  <cand> 7 at token 10232 sub_idx=9 pixel=[324.0,326.0]
+  <cand> 8 at token 10280 sub_idx=9 pixel=[286.0,375.0]
+  <cand> 9 at token 10328 sub_idx=9 pixel=[331.0,346.0]
+  <cand> 0 at token 10842 sub_idx=10 pixel=[317.0,192.0]
+  <cand> 1 at token 10890 sub_idx=10 pixel=[110.0,244.0]
+  <cand> 2 at token 10938 sub_idx=10 pixel=[246.0,181.0]
+  <cand> 3 at token 10986 sub_idx=10 pixel=[209.0,249.0]
+  <cand> 4 at token 11034 sub_idx=10 pixel=[230.0,268.0]
+  <cand> 5 at token 11082 sub_idx=10 pixel=[144.0,222.0]
+  <cand> 6 at token 11130 sub_idx=10 pixel=[276.0,212.0]
+  <cand> 7 at token 11178 sub_idx=10 pixel=[138.0,293.0]
+  <cand> 8 at token 11226 sub_idx=10 pixel=[189.0,298.0]
+  <cand> 9 at token 11274 sub_idx=10 pixel=[150.0,237.0]
+  <cand> 10 at token 11322 sub_idx=10 pixel=[88.0,340.0]
+  <cand> 11 at token 11371 sub_idx=10 pixel=[49.0,330.0]
+  <cand> 12 at token 11420 sub_idx=10 pixel=[128.0,273.0]
+  <cand> 13 at token 11469 sub_idx=10 pixel=[244.0,222.0]
+  <cand> 14 at token 11516 sub_idx=10 pixel=[22.0,204.0]
+  <cand> 15 at token 11565 sub_idx=10 pixel=[223.0,311.0]
+  <cand> 16 at token 11614 sub_idx=10 pixel=[295.0,180.0]
+  <cand> 0 at token 12130 sub_idx=11 pixel=[255.0,239.0]
+  <cand> 1 at token 12178 sub_idx=11 pixel=[129.0,321.0]
+  <cand> 2 at token 12226 sub_idx=11 pixel=[299.0,209.0]
+  <cand> 3 at token 12274 sub_idx=11 pixel=[255.0,210.0]
+  <cand> 4 at token 12322 sub_idx=11 pixel=[109.0,327.0]
+  <cand> 5 at token 12370 sub_idx=11 pixel=[239.0,296.0]
+  <cand> 6 at token 12418 sub_idx=11 pixel=[136.0,291.0]
+  <cand> 7 at token 12466 sub_idx=11 pixel=[211.0,319.0]
+  <cand> 8 at token 12514 sub_idx=11 pixel=[171.0,313.0]
+  <cand> 0 at token 13027 sub_idx=12 pixel=[261.0,348.0]
+  <cand> 1 at token 13075 sub_idx=12 pixel=[321.0,357.0]
+  <cand> 2 at token 13123 sub_idx=12 pixel=[171.0,274.0]
+  <cand> 3 at token 13171 sub_idx=12 pixel=[225.0,182.0]
+  <cand> 4 at token 13219 sub_idx=12 pixel=[270.0,115.0]
+  <cand> 5 at token 13267 sub_idx=12 pixel=[328.0,265.0]
+  <cand> 6 at token 13315 sub_idx=12 pixel=[224.0,161.0]
+  <cand> 0 at token 13828 sub_idx=13 pixel=[221.0,187.0]
+  <cand> 1 at token 13876 sub_idx=13 pixel=[189.0,204.0]
+  <cand> 2 at token 13924 sub_idx=13 pixel=[220.0,160.0]
+  <cand> 3 at token 13972 sub_idx=13 pixel=[194.0,236.0]
+  <cand> 4 at token 14020 sub_idx=13 pixel=[150.0,247.0]
+  <cand> 5 at token 14068 sub_idx=13 pixel=[210.0,197.0]
+  <cand> 0 at token 14582 sub_idx=14 pixel=[219.0,191.0]
+  <cand> 1 at token 14630 sub_idx=14 pixel=[150.0,237.0]
+  <cand> 2 at token 14678 sub_idx=14 pixel=[266.0,213.0]
+  <cand> 3 at token 14726 sub_idx=14 pixel=[196.0,129.0]
+  <cand> 4 at token 14774 sub_idx=14 pixel=[213.0,347.0]
+  <cand> 5 at token 14822 sub_idx=14 pixel=[173.0,265.0]
+  <cand> 6 at token 14870 sub_idx=14 pixel=[215.0,274.0]
+  <cand> 7 at token 14918 sub_idx=14 pixel=[165.0,210.0]
+  <cand> 8 at token 14966 sub_idx=14 pixel=[264.0,279.0]
+  <cand> 9 at token 15014 sub_idx=14 pixel=[155.0,191.0]
+  <cand> 10 at token 15062 sub_idx=14 pixel=[373.0,380.0]
+  <cand> 11 at token 15111 sub_idx=14 pixel=[136.0,265.0]
+  <cand> 12 at token 15160 sub_idx=14 pixel=[223.0,259.0]
+  <cand> 13 at token 15209 sub_idx=14 pixel=[292.0,249.0]
+  <cand> 14 at token 15258 sub_idx=14 pixel=[357.0,387.0]
+  <cand> 0 at token 15774 sub_idx=15 pixel=[249.0,86.0]
+  <cand> 1 at token 15822 sub_idx=15 pixel=[173.0,218.0]
+  <cand> 2 at token 15870 sub_idx=15 pixel=[280.0,249.0]
+  <cand> 3 at token 15918 sub_idx=15 pixel=[288.0,211.0]
+  <cand> 4 at token 15966 sub_idx=15 pixel=[228.0,82.0]
+  <cand> 5 at token 16014 sub_idx=15 pixel=[310.0,161.0]
+  <cand> 6 at token 16062 sub_idx=15 pixel=[178.0,234.0]
+  <cand> 7 at token 16110 sub_idx=15 pixel=[232.0,51.0]
+  <cand> 0 at token 16624 sub_idx=16 pixel=[104.0,112.0]
+  <cand> 1 at token 16672 sub_idx=16 pixel=[182.0,47.0]
+  <cand> 2 at token 16720 sub_idx=16 pixel=[189.0,245.0]
+  <cand> 3 at token 16768 sub_idx=16 pixel=[245.0,57.0]
+  <cand> 4 at token 16816 sub_idx=16 pixel=[325.0,61.0]
+  <cand> 5 at token 16864 sub_idx=16 pixel=[108.0,230.0]
+  <cand> 6 at token 16912 sub_idx=16 pixel=[98.0,101.0]
+  <cand> 7 at token 16960 sub_idx=16 pixel=[365.0,224.0]
+  <cand> 8 at token 17008 sub_idx=16 pixel=[223.0,175.0]
+  <cand> 9 at token 17056 sub_idx=16 pixel=[260.0,311.0]
+  <cand> 10 at token 17104 sub_idx=16 pixel=[216.0,251.0]
+  <cand> 11 at token 17151 sub_idx=16 pixel=[284.0,246.0]
+  <cand> 12 at token 17200 sub_idx=16 pixel=[206.0,195.0]
+  <cand> 13 at token 17249 sub_idx=16 pixel=[339.0,258.0]
+  <cand> 14 at token 17298 sub_idx=16 pixel=[302.0,249.0]
+  <cand> 15 at token 17347 sub_idx=16 pixel=[316.0,177.0]
+  <cand> 16 at token 17396 sub_idx=16 pixel=[271.0,154.0]
+  <cand> 0 at token 17912 sub_idx=17 pixel=[281.0,280.0]
+  <cand> 1 at token 17960 sub_idx=17 pixel=[250.0,245.0]
+  <cand> 2 at token 18008 sub_idx=17 pixel=[197.0,321.0]
+  <cand> 3 at token 18056 sub_idx=17 pixel=[188.0,271.0]
+  <cand> 4 at token 18104 sub_idx=17 pixel=[383.0,250.0]
+  <cand> 5 at token 18152 sub_idx=17 pixel=[363.0,189.0]
+  <cand> 6 at token 18200 sub_idx=17 pixel=[206.0,180.0]
+  <cand> 7 at token 18248 sub_idx=17 pixel=[305.0,327.0]
+  <cand> 8 at token 18296 sub_idx=17 pixel=[244.0,157.0]
+  <cand> 9 at token 18344 sub_idx=17 pixel=[206.0,128.0]
+  <cand> 10 at token 18392 sub_idx=17 pixel=[365.0,172.0]
+  <cand> 11 at token 18441 sub_idx=17 pixel=[174.0,295.0]
+  <cand> 12 at token 18490 sub_idx=17 pixel=[355.0,309.0]
+  <cand> 13 at token 18539 sub_idx=17 pixel=[240.0,235.0]
+  <cand> 14 at token 18586 sub_idx=17 pixel=[153.0,326.0]
+  <cand> 15 at token 18635 sub_idx=17 pixel=[238.0,325.0]
+  <cand> 16 at token 18684 sub_idx=17 pixel=[364.0,327.0]
+  <cand> 17 at token 18733 sub_idx=17 pixel=[179.0,248.0]
+  <cand> 0 at token 19250 sub_idx=18 pixel=[196.0,286.0]
+  <cand> 1 at token 19298 sub_idx=18 pixel=[186.0,308.0]
+  <cand> 2 at token 19346 sub_idx=18 pixel=[354.0,318.0]
+  <cand> 3 at token 19394 sub_idx=18 pixel=[196.0,186.0]
+  <cand> 4 at token 19442 sub_idx=18 pixel=[212.0,193.0]
+  <cand> 5 at token 19490 sub_idx=18 pixel=[205.0,259.0]
+  <cand> 6 at token 19538 sub_idx=18 pixel=[321.0,249.0]
+  <cand> 7 at token 19586 sub_idx=18 pixel=[236.0,238.0]
+  <cand> 8 at token 19634 sub_idx=18 pixel=[323.0,267.0]
+  <cand> 9 at token 19682 sub_idx=18 pixel=[152.0,188.0]
+  <cand> 10 at token 19730 sub_idx=18 pixel=[305.0,190.0]
+  <cand> 11 at token 19779 sub_idx=18 pixel=[237.0,268.0]
+  <cand> 12 at token 19826 sub_idx=18 pixel=[143.0,203.0]
+  <cand> 13 at token 19875 sub_idx=18 pixel=[226.0,280.0]
+  <cand> 14 at token 19924 sub_idx=18 pixel=[237.0,320.0]
+  <cand> 15 at token 19973 sub_idx=18 pixel=[188.0,183.0]
+  <cand> 0 at token 20489 sub_idx=19 pixel=[197.0,269.0]
+  <cand> 1 at token 20537 sub_idx=19 pixel=[227.0,291.0]
+  <cand> 2 at token 20585 sub_idx=19 pixel=[209.0,204.0]
+  <cand> 3 at token 20633 sub_idx=19 pixel=[230.0,264.0]
+  <cand> 4 at token 20681 sub_idx=19 pixel=[247.0,174.0]
+  <cand> 5 at token 20729 sub_idx=19 pixel=[246.0,194.0]
+  <cand> 6 at token 20777 sub_idx=19 pixel=[264.0,226.0]
+  <cand> 7 at token 20825 sub_idx=19 pixel=[260.0,217.0]
+  <cand> 0 at token 21338 sub_idx=20 pixel=[237.0,243.0]
+  <cand> 1 at token 21384 sub_idx=20 pixel=[277.0,77.0]
+  <cand> 2 at token 21432 sub_idx=20 pixel=[310.0,237.0]
+  <cand> 3 at token 21480 sub_idx=20 pixel=[328.0,289.0]
+  <cand> 4 at token 21528 sub_idx=20 pixel=[245.0,101.0]
+  <cand> 5 at token 21576 sub_idx=20 pixel=[241.0,159.0]
+  <cand> 6 at token 21624 sub_idx=20 pixel=[183.0,200.0]
+  <cand> 7 at token 21672 sub_idx=20 pixel=[229.0,270.0]
+  <cand> 0 at token 22187 sub_idx=21 pixel=[134.0,124.0]
+  <cand> 1 at token 22235 sub_idx=21 pixel=[195.0,137.0]
+  <cand> 2 at token 22283 sub_idx=21 pixel=[65.0,112.0]
+  <cand> 3 at token 22331 sub_idx=21 pixel=[255.0,264.0]
+  <cand> 4 at token 22379 sub_idx=21 pixel=[207.0,281.0]
+  <cand> 5 at token 22427 sub_idx=21 pixel=[97.0,122.0]
+  <cand> 6 at token 22475 sub_idx=21 pixel=[241.0,182.0]
+  <cand> 7 at token 22523 sub_idx=21 pixel=[226.0,183.0]
+  <cand> 0 at token 23037 sub_idx=22 pixel=[242.0,288.0]
+  <cand> 1 at token 23085 sub_idx=22 pixel=[113.0,419.0]
+  <cand> 2 at token 23133 sub_idx=22 pixel=[218.0,220.0]
+  <cand> 3 at token 23179 sub_idx=22 pixel=[142.0,145.0]
+  <cand> 4 at token 23227 sub_idx=22 pixel=[218.0,368.0]
+  <cand> 5 at token 23275 sub_idx=22 pixel=[208.0,266.0]
+  <cand> 6 at token 23323 sub_idx=22 pixel=[198.0,377.0]
+  <cand> 7 at token 23371 sub_idx=22 pixel=[122.0,372.0]
+  <cand> 8 at token 23419 sub_idx=22 pixel=[226.0,278.0]
+  <cand> 9 at token 23467 sub_idx=22 pixel=[239.0,278.0]
+  <cand> 0 at token 23980 sub_idx=23 pixel=[197.0,39.0]
+  <cand> 1 at token 24028 sub_idx=23 pixel=[290.0,274.0]
+  <cand> 2 at token 24076 sub_idx=23 pixel=[228.0,337.0]
+  <cand> 3 at token 24124 sub_idx=23 pixel=[259.0,296.0]
+  <cand> 4 at token 24172 sub_idx=23 pixel=[267.0,247.0]
+  <cand> 5 at token 24220 sub_idx=23 pixel=[185.0,32.0]
+  <cand> 6 at token 24268 sub_idx=23 pixel=[318.0,213.0]
+  <cand> 7 at token 24316 sub_idx=23 pixel=[316.0,38.0]
+  <cand> 8 at token 24364 sub_idx=23 pixel=[267.0,316.0]
+  <cand> 9 at token 24412 sub_idx=23 pixel=[209.0,218.0]
+  <e_cand> 0 at token 490 sub_idx=0 (same embed)
+  <e_cand> 1 at token 538 sub_idx=0 (same embed)
+  <e_cand> 2 at token 586 sub_idx=0 (same embed)
+  <e_cand> 0 at token 1974 sub_idx=1 (same embed)
+  <e_cand> 1 at token 2022 sub_idx=1 (same embed)
+  <e_cand> 2 at token 2070 sub_idx=1 (same embed)
+  <e_cand> 0 at token 3213 sub_idx=2 (same embed)
+  <e_cand> 1 at token 3261 sub_idx=2 (same embed)
+  <e_cand> 2 at token 3309 sub_idx=2 (same embed)
+  <e_cand> 0 at token 4598 sub_idx=3 (same embed)
+  <e_cand> 1 at token 4646 sub_idx=3 (same embed)
+  <e_cand> 2 at token 4694 sub_idx=3 (same embed)
+  <e_cand> 0 at token 5546 sub_idx=4 (same embed)
+  <e_cand> 1 at token 5594 sub_idx=4 (same embed)
+  <e_cand> 2 at token 5642 sub_idx=4 (same embed)
+  <e_cand> 0 at token 6640 sub_idx=5 (same embed)
+  <e_cand> 1 at token 6688 sub_idx=5 (same embed)
+  <e_cand> 2 at token 6736 sub_idx=5 (same embed)
+  <e_cand> 0 at token 7537 sub_idx=6 (same embed)
+  <e_cand> 1 at token 7585 sub_idx=6 (same embed)
+  <e_cand> 2 at token 7633 sub_idx=6 (same embed)
+  <e_cand> 0 at token 8194 sub_idx=7 (same embed)
+  <e_cand> 1 at token 8242 sub_idx=7 (same embed)
+  <e_cand> 2 at token 8290 sub_idx=7 (same embed)
+  <e_cand> 0 at token 8998 sub_idx=8 (same embed)
+  <e_cand> 1 at token 9046 sub_idx=8 (same embed)
+  <e_cand> 2 at token 9094 sub_idx=8 (same embed)
+  <e_cand> 0 at token 9944 sub_idx=9 (same embed)
+  <e_cand> 1 at token 9992 sub_idx=9 (same embed)
+  <e_cand> 2 at token 10040 sub_idx=9 (same embed)
+  <e_cand> 0 at token 10888 sub_idx=10 (same embed)
+  <e_cand> 1 at token 10936 sub_idx=10 (same embed)
+  <e_cand> 2 at token 10984 sub_idx=10 (same embed)
+  <e_cand> 0 at token 12176 sub_idx=11 (same embed)
+  <e_cand> 1 at token 12224 sub_idx=11 (same embed)
+  <e_cand> 2 at token 12272 sub_idx=11 (same embed)
+  <e_cand> 0 at token 13073 sub_idx=12 (same embed)
+  <e_cand> 1 at token 13121 sub_idx=12 (same embed)
+  <e_cand> 2 at token 13169 sub_idx=12 (same embed)
+  <e_cand> 0 at token 13874 sub_idx=13 (same embed)
+  <e_cand> 1 at token 13922 sub_idx=13 (same embed)
+  <e_cand> 2 at token 13970 sub_idx=13 (same embed)
+  <e_cand> 0 at token 14628 sub_idx=14 (same embed)
+  <e_cand> 1 at token 14676 sub_idx=14 (same embed)
+  <e_cand> 2 at token 14724 sub_idx=14 (same embed)
+  <e_cand> 0 at token 15820 sub_idx=15 (same embed)
+  <e_cand> 1 at token 15868 sub_idx=15 (same embed)
+  <e_cand> 2 at token 15916 sub_idx=15 (same embed)
+  <e_cand> 0 at token 16670 sub_idx=16 (same embed)
+  <e_cand> 1 at token 16718 sub_idx=16 (same embed)
+  <e_cand> 2 at token 16766 sub_idx=16 (same embed)
+  <e_cand> 0 at token 17958 sub_idx=17 (same embed)
+  <e_cand> 1 at token 18006 sub_idx=17 (same embed)
+  <e_cand> 2 at token 18054 sub_idx=17 (same embed)
+  <e_cand> 0 at token 19296 sub_idx=18 (same embed)
+  <e_cand> 1 at token 19344 sub_idx=18 (same embed)
+  <e_cand> 2 at token 19392 sub_idx=18 (same embed)
+  <e_cand> 0 at token 20535 sub_idx=19 (same embed)
+  <e_cand> 1 at token 20583 sub_idx=19 (same embed)
+  <e_cand> 2 at token 20631 sub_idx=19 (same embed)
+  <e_cand> 0 at token 21382 sub_idx=20 (same embed)
+  <e_cand> 1 at token 21430 sub_idx=20 (same embed)
+  <e_cand> 2 at token 21478 sub_idx=20 (same embed)
+  <e_cand> 0 at token 22233 sub_idx=21 (same embed)
+  <e_cand> 1 at token 22281 sub_idx=21 (same embed)
+  <e_cand> 2 at token 22329 sub_idx=21 (same embed)
+  <e_cand> 0 at token 23083 sub_idx=22 (same embed)
+  <e_cand> 1 at token 23131 sub_idx=22 (same embed)
+  <e_cand> 2 at token 23177 sub_idx=22 (same embed)
+  <e_cand> 0 at token 24026 sub_idx=23 (same embed)
+  <e_cand> 1 at token 24074 sub_idx=23 (same embed)
+  <e_cand> 2 at token 24122 sub_idx=23 (same embed)

wandb/run-20260225_063717-0ub00jhc/files/requirements.txt

@@ -0,0 +1,113 @@
+nvidia-cuda-runtime-cu12==12.1.105
+platformdirs==4.9.2
+wandb==0.22.3
+transformers==4.37.2
+einops==0.8.2
+accelerate==0.33.0
+aiosignal==1.4.0
+huggingface_hub==0.36.2
+pycparser==2.22
+multidict==6.7.1
+urllib3==2.6.3
+urllib3==2.5.0
+aiohappyeyeballs==2.6.1
+nvidia-cusolver-cu12==11.4.5.107
+yarl==1.22.0
+typing-inspection==0.4.2
+gitdb==4.0.12
+multiprocess==0.70.16
+async-timeout==5.0.1
+Jinja2==3.1.6
+setuptools==82.0.0
+torch==2.5.1+cu121
+hyperframe==6.1.0
+peft==0.10.0
+tzdata==2025.3
+dill==0.3.8
+orjson==3.11.7
+nvidia-cufft-cu12==11.0.2.54
+propcache==0.4.1
+PySocks==1.7.1
+nvidia-cublas-cu12==12.1.3.1
+nvidia-nccl-cu12==2.21.5
+docker-pycreds==0.4.0
+pydantic_core==2.41.5
+nvidia-ml-py==13.590.48
+ninja==1.13.0
+cffi==1.15.0
+nvidia-cuda-cupti-cu12==12.1.105
+triton==3.1.0
+annotated-types==0.7.0
+nvidia-nvtx-cu12==12.1.105
+timm==1.0.22
+sympy==1.13.1
+pydantic==2.12.5
+xxhash==3.6.0
+py-cpuinfo==9.0.0
+sentry-sdk==2.53.0
+networkx==3.4.2
+click==8.3.1
+regex==2026.1.15
+pillow==12.0.0
+zstandard==0.23.0
+pyarrow==20.0.0
+GitPython==3.1.46
+wheel==0.46.3
+fsspec==2025.3.0
+typing_extensions==4.15.0
+hf-xet==1.2.0
+torchvision==0.20.1+cu121
+python-dateutil==2.9.0.post0
+numpy==1.26.4
+smmap==5.0.2
+setproctitle==1.2.3
+safetensors==0.7.0
+requests==2.32.5
+hpack==4.1.0
+six==1.17.0
+modelscope==1.34.0
+pip==26.0.1
+h2==4.3.0
+PyYAML==6.0.3
+PyYAML==6.0
+torchaudio==2.5.1+cu121
+nvidia-nvjitlink-cu12==12.9.86
+psutil==7.2.2
+psutil==5.9.1
+pandas==2.3.2
+pytz==2025.2
+nvidia-cudnn-cu12==9.1.0.70
+packaging==25.0
+hjson==3.1.0
+deepspeed==0.14.4
+datasets==4.0.0
+nvidia-cuda-nvrtc-cu12==12.1.105
+tokenizers==0.15.2
+MarkupSafe==2.1.5
+charset-normalizer==3.4.4
+frozenlist==1.8.0
+Brotli==1.0.9
+certifi==2026.1.4
+mpmath==1.3.0
+protobuf==3.20.1
+tqdm==4.67.3
+nvidia-cusparse-cu12==12.1.0.106
+attrs==25.4.0
+appdirs==1.4.4
+flash_attn==2.7.4.post1
+aiohttp==3.13.3
+filelock==3.20.0
+idna==3.11
+nvidia-curand-cu12==10.3.2.106
+zipp==3.23.0
+backports.tarfile==1.2.0
+autocommand==2.2.2
+platformdirs==4.4.0
+jaraco.text==4.0.0
+more-itertools==10.8.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+wheel==0.46.3
+tomli==2.4.0
+packaging==26.0
+importlib_metadata==8.7.1

wandb/run-20260225_063717-0ub00jhc/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-3.10.0-957.el7.x86_64-x86_64-with-glibc2.17",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2026-02-24T22:37:17.678092Z",
+  "args":  [
+    "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/resolved_train_config.json"
+  ],
+  "program":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py",
+  "codePath":  "InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py",
+  "codePathLocal":  "internvl_cleaned/train/internvl_chat_finetune.py",
+  "git":  {
+    "remote":  "git@github.com:Yuxin916/CL_CoTNav.git",
+    "commit":  "f7d6fbe6a8031d29a95c2f3be79e99f96670a12e"
+  },
+  "email":  "caiy0039@e.ntu.edu.sg",
+  "root":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY",
+  "host":  "SH-IDC1-10-140-37-45",
+  "executable":  "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/bin/python3.10",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "524945911808",
+      "used":  "39460663296"
+    }
+  },
+  "memory":  {
+    "total":  "1081627828224"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-9e78ad4b-b304-a199-38f4-24b3acd9b531"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-19c9429b-5ecf-0b76-7d81-03c36a449f32"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-57520d78-d04a-3028-bd40-5ebd78e123e6"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-0d94270a-8ad3-b3a6-4acd-145fccc36d85"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-b9b6fe80-e37a-b622-0deb-27ef46a965ff"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-7b870038-dbe6-4039-b6bf-f90517f43f6c"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-e3065d01-9539-64c9-417f-91273b521051"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-28ce6eed-85b8-155a-5a5d-412dfb1dd1c0"
+    }
+  ],
+  "cudaVersion":  "12.2",
+  "slurm":  {
+    "cluster_name":  "cluster_sproject3",
+    "conf":  "/etc/slurm/slurm.conf",
+    "cpus_on_node":  "128",
+    "cpus_per_task":  "8",
+    "distribution":  "cyclic",
+    "gtids":  "0",
+    "job_account":  "research",
+    "job_cpus_per_node":  "128",
+    "job_cpus_per_node_pack_group_0":  "128",
+    "job_gid":  "200000139",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "7464696",
+    "job_name":  "vlm_ft",
+    "job_nodelist":  "SH-IDC1-10-140-37-45",
+    "job_num_nodes":  "1",
+    "job_partition":  "interntmp",
+    "job_qos":  "normal",
+    "job_uid":  "200000139",
+    "job_user":  "wangmaonan",
+    "jobid":  "7464696",
+    "launch_node_ipaddr":  "10.140.37.45",
+    "localid":  "0",
+    "mem_per_node":  "49152",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "SH-IDC1-10-140-37-45",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "srun_comm_host":  "10.140.37.45",
+    "srun_comm_port":  "40413",
+    "step_gpus":  "0,1,2,3",
+    "step_id":  "0",
+    "step_launcher_port":  "40413",
+    "step_nodelist":  "SH-IDC1-10-140-37-45",
+    "step_num_nodes":  "1",
+    "step_num_tasks":  "1",
+    "step_tasks_per_node":  "1",
+    "stepid":  "0",
+    "submit_dir":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav",
+    "submit_host":  "SH-IDC1-10-140-37-45",
+    "task_pid":  "236139",
+    "tasks_per_node":  "1",
+    "topology_addr":  "SH-IDC1-10-140-37-45",
+    "topology_addr_pattern":  "node",
+    "umask":  "0002",
+    "working_cluster":  "cluster_sproject3:SH-IDC1-10-140-37-161:6817:9216:109"
+  },
+  "writerId":  "1xd6l9g7kzysn4yq7wybuhrbyoibnbxf"
+}

wandb/run-20260225_063717-0ub00jhc/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2026-02-25T06:37:17.85489302+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmps47xgacm/port-236721.txt","pid":236721,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-02-25T06:37:17.857743824+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":236721}
+{"time":"2026-02-25T06:37:17.857993952+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-236721-239969-3041544036/socket","Net":"unix"}}
+{"time":"2026-02-25T06:37:17.925755794+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-02-25T06:37:17.936203006+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"0ub00jhc","id":"1(@)"}
+{"time":"2026-02-25T06:37:18.342592105+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"0ub00jhc","id":"1(@)"}
+{"time":"2026-02-25T06:54:14.629396898+08:00","level":"INFO","msg":"server: parent process exited, terminating service process"}

wandb/run-20260225_063717-0ub00jhc/logs/debug-internal.log

@@ -0,0 +1,6 @@
+{"time":"2026-02-25T06:37:17.937232022+08:00","level":"INFO","msg":"stream: starting","core version":"0.22.3"}
+{"time":"2026-02-25T06:37:18.338623764+08:00","level":"INFO","msg":"stream: created new stream","id":"0ub00jhc"}
+{"time":"2026-02-25T06:37:18.339228707+08:00","level":"INFO","msg":"handler: started","stream_id":"0ub00jhc"}
+{"time":"2026-02-25T06:37:18.342133656+08:00","level":"INFO","msg":"stream: started","id":"0ub00jhc"}
+{"time":"2026-02-25T06:37:18.34215159+08:00","level":"INFO","msg":"sender: started","stream_id":"0ub00jhc"}
+{"time":"2026-02-25T06:37:18.342148878+08:00","level":"INFO","msg":"writer: started","stream_id":"0ub00jhc"}

wandb/run-20260225_063717-0ub00jhc/logs/debug.log

@@ -0,0 +1,22 @@
+2026-02-25 06:37:17,697 INFO    MainThread:236721 [wandb_setup.py:_flush():81] Current SDK version is 0.22.3
+2026-02-25 06:37:17,699 INFO    MainThread:236721 [wandb_setup.py:_flush():81] Configure stats pid to 236721
+2026-02-25 06:37:17,699 INFO    MainThread:236721 [wandb_setup.py:_flush():81] Loading settings from /mnt/petrelfs/wangmaonan/.config/wandb/settings
+2026-02-25 06:37:17,700 INFO    MainThread:236721 [wandb_setup.py:_flush():81] Loading settings from /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/wandb/settings
+2026-02-25 06:37:17,700 INFO    MainThread:236721 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-02-25 06:37:17,701 INFO    MainThread:236721 [wandb_init.py:setup_run_log_directory():706] Logging user logs to /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/wandb/run-20260225_063717-0ub00jhc/logs/debug.log
+2026-02-25 06:37:17,701 INFO    MainThread:236721 [wandb_init.py:setup_run_log_directory():707] Logging internal logs to /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/wandb/run-20260225_063717-0ub00jhc/logs/debug-internal.log
+2026-02-25 06:37:17,702 INFO    MainThread:236721 [wandb_init.py:init():833] calling init triggers
+2026-02-25 06:37:17,702 INFO    MainThread:236721 [wandb_init.py:init():838] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-02-25 06:37:17,703 INFO    MainThread:236721 [wandb_init.py:init():881] starting backend
+2026-02-25 06:37:17,925 INFO    MainThread:236721 [wandb_init.py:init():884] sending inform_init request
+2026-02-25 06:37:17,932 INFO    MainThread:236721 [wandb_init.py:init():892] backend started and connected
+2026-02-25 06:37:17,933 INFO    MainThread:236721 [wandb_init.py:init():962] updated telemetry
+2026-02-25 06:37:17,963 INFO    MainThread:236721 [wandb_init.py:init():986] communicating run to backend with 90.0 second timeout
+2026-02-25 06:37:19,017 INFO    MainThread:236721 [wandb_init.py:init():1033] starting run threads in backend
+2026-02-25 06:37:19,332 INFO    MainThread:236721 [wandb_run.py:_console_start():2506] atexit reg
+2026-02-25 06:37:19,333 INFO    MainThread:236721 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2026-02-25 06:37:19,333 INFO    MainThread:236721 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2026-02-25 06:37:19,333 INFO    MainThread:236721 [wandb_run.py:_redirect():2446] Redirects installed.
+2026-02-25 06:37:19,340 INFO    MainThread:236721 [wandb_init.py:init():1073] run started, returning control to user process
+2026-02-25 06:37:19,343 INFO    MainThread:236721 [wandb_run.py:_config_callback():1390] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'torch.bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['InternVLChatModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '../pretrained/InternVL3-2B', '_commit_hash': None, '_attn_implementation_internal': None, 'transformers_version': None, 'auto_map': {'AutoConfig': 'configuration_internvl_chat.InternVLChatConfig', 'AutoModel': 'modeling_internvl_chat.InternVLChatModel', 'AutoModelForCausalLM': 'modeling_internvl_chat.InternVLChatModel'}, 'hidden_size': 1536, 'image_fold': None, 'model_type': 'internvl_chat', 'system_message': 'You are an autonomous navigation agent operating in indoor environments. You receive spatial information through position embeddings injected into visual features and text tokens. Use the BEV map, position embeddings, and semantic information to make navigation decisions. When the target object is detected (<target> marker), navigate directly to it. Otherwise, explore frontiers strategically to find the goal object.', 'vision_config': {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': True, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['InternVisionModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'OpenGVLab/InternViT-6B-448px-V1-5', 'transformers_version': '4.37.2', '_attn_implementation_autoset': True, 'auto_map': {'AutoConfig': 'configuration_intern_vit.InternVisionConfig', 'AutoModel': 'modeling_intern_vit.InternVisionModel'}, 'capacity_factor': 1.2, 'eval_capacity_factor': 1.4, 'laux_allreduce': 'all_nodes', 'model_type': 'intern_vit_6b', 'moe_coeff_ratio': 0.5, 'moe_intermediate_size': 768, 'moe_output_scale': 4.0, 'noisy_gate_policy': 'RSample_before', 'num_experts': 8, 'num_routed_experts': 4, 'num_shared_experts': 4, 'shared_expert_intermediate_size': 3072, 'use_moe': False, 'use_residual': True, 'use_rts': False, 'use_weighted_residual': False, 'hidden_size': 1024, 'intermediate_size': 4096, 'dropout': 0.0, 'drop_path_rate': 0.0, 'num_hidden_layers': 24, 'num_attention_heads': 16, 'num_channels': 3, 'patch_size': 14, 'image_size': 448, 'initializer_range': 1e-10, 'initializer_factor': 0.1, 'attention_dropout': 0.0, 'layer_norm_eps': 1e-06, 'hidden_act': 'gelu', 'norm_type': 'layer_norm', 'qkv_bias': True, 'qk_normalization': False, 'use_flash_attn': True}, 'llm_config': {'vocab_size': 151677, 'max_position_embeddings': 32768, 'hidden_size': 1536, 'intermediate_size': 8960, 'num_hidden_layers': 28, 'num_attention_heads': 12, 'use_sliding_window': False, 'sliding_window': None, 'max_window_layers': 70, 'num_key_value_heads': 2, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 1000000.0, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': True, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['Qwen2ForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 151643, 'pad_token_id': None, 'eos_token_id': 151643, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': './pretrained/Qwen2.5-32B-Instruct', 'transformers_version': '4.37.2', '_attn_implementation_autoset': True, 'model_type': 'qwen2', 'moe_config': None, 'rope_scaling': {'factor': 2.0, 'rope_type': 'dynamic', 'type': 'dynamic'}, 'attn_implementation': 'flash_attention_2'}, 'use_backbone_lora': 0, 'use_llm_lora': 64, 'pad2square': False, 'select_layer': -1, 'force_image_size': 448, 'downsample_ratio': 0.5, 'template': 'internvl2_5_nav', 'dynamic_image_size': False, 'use_thumbnail': True, 'ps_version': 'v2', 'min_dynamic_patch': 1, 'max_dynamic_patch': 12, 'num_image_token_bev': 256, 'num_image_token_ego': 32, 'use_pairwise_spatial_encoder': False, 'use_position_embeddings': True, 'dual_text_pos_injection': True, 'bev_image_size': 448, 'vit_bev_freeze': True, 'vit_bev_use_lora': True, 'vit_bev_lora_rank': 64, 'vit_rgb_freeze': True, 'vit_rgb_use_lora': True, 'vit_rgb_lora_rank': 16, 'output_dir': '/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': 9300, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'runs/Feb25_06-36-26_SH-IDC1-10-140-37-45', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 0.5, 'save_total_limit': 2, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 12, 'past_index': -1, 'run_name': 'a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY_steps9300_gpus4_acc1', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'deepspeed': 'zero_stage2_config_acc1.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': False, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None}

wandb/run-20260225_063717-0ub00jhc/run-0ub00jhc.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e269023c47e74756d9b57ae41c89068739f24c06f6ea070aa7b3e56a496b14df
+size 262144

wandb/run-20260225_094307-hmhb8ltr/files/requirements.txt

@@ -0,0 +1,113 @@
+nvidia-cuda-runtime-cu12==12.1.105
+platformdirs==4.9.2
+wandb==0.22.3
+transformers==4.37.2
+einops==0.8.2
+accelerate==0.33.0
+aiosignal==1.4.0
+huggingface_hub==0.36.2
+pycparser==2.22
+multidict==6.7.1
+urllib3==2.6.3
+urllib3==2.5.0
+aiohappyeyeballs==2.6.1
+nvidia-cusolver-cu12==11.4.5.107
+yarl==1.22.0
+typing-inspection==0.4.2
+gitdb==4.0.12
+multiprocess==0.70.16
+async-timeout==5.0.1
+Jinja2==3.1.6
+setuptools==82.0.0
+torch==2.5.1+cu121
+hyperframe==6.1.0
+peft==0.10.0
+tzdata==2025.3
+dill==0.3.8
+orjson==3.11.7
+nvidia-cufft-cu12==11.0.2.54
+propcache==0.4.1
+PySocks==1.7.1
+nvidia-cublas-cu12==12.1.3.1
+nvidia-nccl-cu12==2.21.5
+docker-pycreds==0.4.0
+pydantic_core==2.41.5
+nvidia-ml-py==13.590.48
+ninja==1.13.0
+cffi==1.15.0
+nvidia-cuda-cupti-cu12==12.1.105
+triton==3.1.0
+annotated-types==0.7.0
+nvidia-nvtx-cu12==12.1.105
+timm==1.0.22
+sympy==1.13.1
+pydantic==2.12.5
+xxhash==3.6.0
+py-cpuinfo==9.0.0
+sentry-sdk==2.53.0
+networkx==3.4.2
+click==8.3.1
+regex==2026.1.15
+pillow==12.0.0
+zstandard==0.23.0
+pyarrow==20.0.0
+GitPython==3.1.46
+wheel==0.46.3
+fsspec==2025.3.0
+typing_extensions==4.15.0
+hf-xet==1.2.0
+torchvision==0.20.1+cu121
+python-dateutil==2.9.0.post0
+numpy==1.26.4
+smmap==5.0.2
+setproctitle==1.2.3
+safetensors==0.7.0
+requests==2.32.5
+hpack==4.1.0
+six==1.17.0
+modelscope==1.34.0
+pip==26.0.1
+h2==4.3.0
+PyYAML==6.0.3
+PyYAML==6.0
+torchaudio==2.5.1+cu121
+nvidia-nvjitlink-cu12==12.9.86
+psutil==7.2.2
+psutil==5.9.1
+pandas==2.3.2
+pytz==2025.2
+nvidia-cudnn-cu12==9.1.0.70
+packaging==25.0
+hjson==3.1.0
+deepspeed==0.14.4
+datasets==4.0.0
+nvidia-cuda-nvrtc-cu12==12.1.105
+tokenizers==0.15.2
+MarkupSafe==2.1.5
+charset-normalizer==3.4.4
+frozenlist==1.8.0
+Brotli==1.0.9
+certifi==2026.1.4
+mpmath==1.3.0
+protobuf==3.20.1
+tqdm==4.67.3
+nvidia-cusparse-cu12==12.1.0.106
+attrs==25.4.0
+appdirs==1.4.4
+flash_attn==2.7.4.post1
+aiohttp==3.13.3
+filelock==3.20.0
+idna==3.11
+nvidia-curand-cu12==10.3.2.106
+zipp==3.23.0
+backports.tarfile==1.2.0
+autocommand==2.2.2
+platformdirs==4.4.0
+jaraco.text==4.0.0
+more-itertools==10.8.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+wheel==0.46.3
+tomli==2.4.0
+packaging==26.0
+importlib_metadata==8.7.1

wandb/run-20260225_094307-hmhb8ltr/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-3.10.0-957.el7.x86_64-x86_64-with-glibc2.17",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2026-02-25T01:43:07.009034Z",
+  "args":  [
+    "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/resolved_train_config.json"
+  ],
+  "program":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py",
+  "codePath":  "InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py",
+  "codePathLocal":  "internvl_cleaned/train/internvl_chat_finetune.py",
+  "git":  {
+    "remote":  "git@github.com:Yuxin916/CL_CoTNav.git",
+    "commit":  "1b65602e555ba143e86856602a3e22083cba01d4"
+  },
+  "email":  "caiy0039@e.ntu.edu.sg",
+  "root":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY",
+  "host":  "SH-IDC1-10-140-37-31",
+  "executable":  "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/bin/python3.10",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "524945911808",
+      "used":  "39849480192"
+    }
+  },
+  "memory":  {
+    "total":  "1081627848704"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-828b568b-c82d-b423-1a34-542c520b513a"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-5e4a67ae-0b1a-fef4-d35c-a727201c09bc"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-0a457f54-5e48-dcbb-8f0e-26fa26366156"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-27a5a9a9-701a-7f08-6cc8-898c967fc7d1"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-778a3a47-8b90-df02-6d77-59c305de90f4"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-d9f06099-a191-6157-a52f-43aa09175792"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-e9da788d-da37-714d-ab0e-c3d8824cf614"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-0c799f3d-966a-4baa-88cd-540cff322767"
+    }
+  ],
+  "cudaVersion":  "12.2",
+  "slurm":  {
+    "cluster_name":  "cluster_sproject3",
+    "conf":  "/etc/slurm/slurm.conf",
+    "cpus_on_node":  "128",
+    "cpus_per_task":  "8",
+    "distribution":  "cyclic",
+    "gtids":  "0",
+    "job_account":  "research",
+    "job_cpus_per_node":  "128",
+    "job_cpus_per_node_pack_group_0":  "128",
+    "job_gid":  "200000139",
+    "job_gpus":  "0,1,2,3,4,5,6,7",
+    "job_id":  "7465487",
+    "job_name":  "vlm_ft",
+    "job_nodelist":  "SH-IDC1-10-140-37-31",
+    "job_num_nodes":  "1",
+    "job_partition":  "interntmp",
+    "job_qos":  "normal",
+    "job_uid":  "200000139",
+    "job_user":  "wangmaonan",
+    "jobid":  "7465487",
+    "launch_node_ipaddr":  "10.140.37.31",
+    "localid":  "0",
+    "mem_per_node":  "49152",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "SH-IDC1-10-140-37-31",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "srun_comm_host":  "10.140.37.31",
+    "srun_comm_port":  "45814",
+    "step_gpus":  "0,1,2,3",
+    "step_id":  "0",
+    "step_launcher_port":  "45814",
+    "step_nodelist":  "SH-IDC1-10-140-37-31",
+    "step_num_nodes":  "1",
+    "step_num_tasks":  "1",
+    "step_tasks_per_node":  "1",
+    "stepid":  "0",
+    "submit_dir":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav",
+    "submit_host":  "SH-IDC1-10-140-37-31",
+    "task_pid":  "188600",
+    "tasks_per_node":  "1",
+    "topology_addr":  "SH-IDC1-10-140-37-31",
+    "topology_addr_pattern":  "node",
+    "umask":  "0002",
+    "working_cluster":  "cluster_sproject3:SH-IDC1-10-140-37-161:6817:9216:109"
+  },
+  "writerId":  "89yi1atsuhvihf32z0oklebiq6eazrep"
+}

wandb/run-20260225_094307-hmhb8ltr/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2026-02-25T09:43:07.172368186+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp9xle39pt/port-189155.txt","pid":189155,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-02-25T09:43:07.17529052+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":189155}
+{"time":"2026-02-25T09:43:07.175438935+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-189155-192110-2041067869/socket","Net":"unix"}}
+{"time":"2026-02-25T09:43:07.249935403+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-02-25T09:43:07.258572841+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"hmhb8ltr","id":"1(@)"}
+{"time":"2026-02-25T09:43:07.683227166+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"hmhb8ltr","id":"1(@)"}
+{"time":"2026-02-25T11:09:21.665063902+08:00","level":"INFO","msg":"server: parent process exited, terminating service process"}

wandb/run-20260225_094307-hmhb8ltr/logs/debug-internal.log

@@ -0,0 +1,14 @@
+{"time":"2026-02-25T09:43:07.25942373+08:00","level":"INFO","msg":"stream: starting","core version":"0.22.3"}
+{"time":"2026-02-25T09:43:07.67936989+08:00","level":"INFO","msg":"stream: created new stream","id":"hmhb8ltr"}
+{"time":"2026-02-25T09:43:07.679921274+08:00","level":"INFO","msg":"handler: started","stream_id":"hmhb8ltr"}
+{"time":"2026-02-25T09:43:07.682825049+08:00","level":"INFO","msg":"stream: started","id":"hmhb8ltr"}
+{"time":"2026-02-25T09:43:07.682829933+08:00","level":"INFO","msg":"writer: started","stream_id":"hmhb8ltr"}
+{"time":"2026-02-25T09:43:07.682846777+08:00","level":"INFO","msg":"sender: started","stream_id":"hmhb8ltr"}
+{"time":"2026-02-25T10:08:08.481616394+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-25T10:08:10.152924176+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":1579}
+{"time":"2026-02-25T10:08:34.648103405+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":29}
+{"time":"2026-02-25T10:09:08.495965349+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-25T10:09:13.630775482+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":1657}
+{"time":"2026-02-25T10:09:28.215288926+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":29}
+{"time":"2026-02-25T10:46:53.782709329+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/tsaisplus-nanyang-technological-university-singapore/prompt_revision/hmhb8ltr/file_stream\": unexpected EOF"}
+{"time":"2026-02-25T10:47:16.973717222+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/tsaisplus-nanyang-technological-university-singapore/prompt_revision/hmhb8ltr/file_stream\": unexpected EOF"}

wandb/run-20260225_094307-hmhb8ltr/logs/debug.log

@@ -0,0 +1,22 @@
+2026-02-25 09:43:07,025 INFO    MainThread:189155 [wandb_setup.py:_flush():81] Current SDK version is 0.22.3
+2026-02-25 09:43:07,026 INFO    MainThread:189155 [wandb_setup.py:_flush():81] Configure stats pid to 189155
+2026-02-25 09:43:07,027 INFO    MainThread:189155 [wandb_setup.py:_flush():81] Loading settings from /mnt/petrelfs/wangmaonan/.config/wandb/settings
+2026-02-25 09:43:07,027 INFO    MainThread:189155 [wandb_setup.py:_flush():81] Loading settings from /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/wandb/settings
+2026-02-25 09:43:07,028 INFO    MainThread:189155 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2026-02-25 09:43:07,028 INFO    MainThread:189155 [wandb_init.py:setup_run_log_directory():706] Logging user logs to /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/wandb/run-20260225_094307-hmhb8ltr/logs/debug.log
+2026-02-25 09:43:07,029 INFO    MainThread:189155 [wandb_init.py:setup_run_log_directory():707] Logging internal logs to /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/wandb/run-20260225_094307-hmhb8ltr/logs/debug-internal.log
+2026-02-25 09:43:07,029 INFO    MainThread:189155 [wandb_init.py:init():833] calling init triggers
+2026-02-25 09:43:07,030 INFO    MainThread:189155 [wandb_init.py:init():838] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2026-02-25 09:43:07,030 INFO    MainThread:189155 [wandb_init.py:init():881] starting backend
+2026-02-25 09:43:07,250 INFO    MainThread:189155 [wandb_init.py:init():884] sending inform_init request
+2026-02-25 09:43:07,256 INFO    MainThread:189155 [wandb_init.py:init():892] backend started and connected
+2026-02-25 09:43:07,257 INFO    MainThread:189155 [wandb_init.py:init():962] updated telemetry
+2026-02-25 09:43:07,284 INFO    MainThread:189155 [wandb_init.py:init():986] communicating run to backend with 90.0 second timeout
+2026-02-25 09:43:08,178 INFO    MainThread:189155 [wandb_init.py:init():1033] starting run threads in backend
+2026-02-25 09:43:08,469 INFO    MainThread:189155 [wandb_run.py:_console_start():2506] atexit reg
+2026-02-25 09:43:08,470 INFO    MainThread:189155 [wandb_run.py:_redirect():2354] redirect: wrap_raw
+2026-02-25 09:43:08,471 INFO    MainThread:189155 [wandb_run.py:_redirect():2423] Wrapping output streams.
+2026-02-25 09:43:08,471 INFO    MainThread:189155 [wandb_run.py:_redirect():2446] Redirects installed.
+2026-02-25 09:43:08,477 INFO    MainThread:189155 [wandb_init.py:init():1073] run started, returning control to user process
+2026-02-25 09:43:08,480 INFO    MainThread:189155 [wandb_run.py:_config_callback():1390] config_cb None None {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'torch.bfloat16', 'use_bfloat16': False, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['InternVLChatModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': '../pretrained/InternVL3-2B', '_commit_hash': None, '_attn_implementation_internal': None, 'transformers_version': None, 'auto_map': {'AutoConfig': 'configuration_internvl_chat.InternVLChatConfig', 'AutoModel': 'modeling_internvl_chat.InternVLChatModel', 'AutoModelForCausalLM': 'modeling_internvl_chat.InternVLChatModel'}, 'hidden_size': 1536, 'image_fold': None, 'model_type': 'internvl_chat', 'system_message': 'You are an autonomous navigation agent operating in indoor environments. You receive spatial information through position embeddings injected into visual features and text tokens. Use the BEV map, position embeddings, and semantic information to make navigation decisions. When the target object is detected (<target> marker), navigate directly to it. Otherwise, explore frontiers strategically to find the goal object.', 'vision_config': {'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': True, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': True, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['InternVisionModel'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': None, 'pad_token_id': None, 'eos_token_id': None, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': 'OpenGVLab/InternViT-6B-448px-V1-5', 'transformers_version': '4.37.2', '_attn_implementation_autoset': True, 'auto_map': {'AutoConfig': 'configuration_intern_vit.InternVisionConfig', 'AutoModel': 'modeling_intern_vit.InternVisionModel'}, 'capacity_factor': 1.2, 'eval_capacity_factor': 1.4, 'laux_allreduce': 'all_nodes', 'model_type': 'intern_vit_6b', 'moe_coeff_ratio': 0.5, 'moe_intermediate_size': 768, 'moe_output_scale': 4.0, 'noisy_gate_policy': 'RSample_before', 'num_experts': 8, 'num_routed_experts': 4, 'num_shared_experts': 4, 'shared_expert_intermediate_size': 3072, 'use_moe': False, 'use_residual': True, 'use_rts': False, 'use_weighted_residual': False, 'hidden_size': 1024, 'intermediate_size': 4096, 'dropout': 0.0, 'drop_path_rate': 0.0, 'num_hidden_layers': 24, 'num_attention_heads': 16, 'num_channels': 3, 'patch_size': 14, 'image_size': 448, 'initializer_range': 1e-10, 'initializer_factor': 0.1, 'attention_dropout': 0.0, 'layer_norm_eps': 1e-06, 'hidden_act': 'gelu', 'norm_type': 'layer_norm', 'qkv_bias': True, 'qk_normalization': False, 'use_flash_attn': True}, 'llm_config': {'vocab_size': 151677, 'max_position_embeddings': 32768, 'hidden_size': 1536, 'intermediate_size': 8960, 'num_hidden_layers': 28, 'num_attention_heads': 12, 'use_sliding_window': False, 'sliding_window': None, 'max_window_layers': 70, 'num_key_value_heads': 2, 'hidden_act': 'silu', 'initializer_range': 0.02, 'rms_norm_eps': 1e-06, 'use_cache': False, 'rope_theta': 1000000.0, 'attention_dropout': 0.0, 'return_dict': True, 'output_hidden_states': False, 'output_attentions': False, 'torchscript': False, 'torch_dtype': 'bfloat16', 'use_bfloat16': True, 'tf_legacy_loss': False, 'pruned_heads': {}, 'tie_word_embeddings': False, 'chunk_size_feed_forward': 0, 'is_encoder_decoder': False, 'is_decoder': False, 'cross_attention_hidden_size': None, 'add_cross_attention': False, 'tie_encoder_decoder': False, 'max_length': 20, 'min_length': 0, 'do_sample': False, 'early_stopping': False, 'num_beams': 1, 'num_beam_groups': 1, 'diversity_penalty': 0.0, 'temperature': 1.0, 'top_k': 50, 'top_p': 1.0, 'typical_p': 1.0, 'repetition_penalty': 1.0, 'length_penalty': 1.0, 'no_repeat_ngram_size': 0, 'encoder_no_repeat_ngram_size': 0, 'bad_words_ids': None, 'num_return_sequences': 1, 'output_scores': False, 'return_dict_in_generate': False, 'forced_bos_token_id': None, 'forced_eos_token_id': None, 'remove_invalid_values': False, 'exponential_decay_length_penalty': None, 'suppress_tokens': None, 'begin_suppress_tokens': None, 'architectures': ['Qwen2ForCausalLM'], 'finetuning_task': None, 'id2label': {0: 'LABEL_0', 1: 'LABEL_1'}, 'label2id': {'LABEL_0': 0, 'LABEL_1': 1}, 'tokenizer_class': None, 'prefix': None, 'bos_token_id': 151643, 'pad_token_id': None, 'eos_token_id': 151643, 'sep_token_id': None, 'decoder_start_token_id': None, 'task_specific_params': None, 'problem_type': None, '_name_or_path': './pretrained/Qwen2.5-32B-Instruct', 'transformers_version': '4.37.2', '_attn_implementation_autoset': True, 'model_type': 'qwen2', 'moe_config': None, 'rope_scaling': {'factor': 2.0, 'rope_type': 'dynamic', 'type': 'dynamic'}, 'attn_implementation': 'flash_attention_2'}, 'use_backbone_lora': 0, 'use_llm_lora': 64, 'pad2square': False, 'select_layer': -1, 'force_image_size': 448, 'downsample_ratio': 0.5, 'template': 'internvl2_5_nav', 'dynamic_image_size': False, 'use_thumbnail': True, 'ps_version': 'v2', 'min_dynamic_patch': 1, 'max_dynamic_patch': 12, 'num_image_token_bev': 256, 'num_image_token_ego': 32, 'use_pairwise_spatial_encoder': False, 'use_position_embeddings': True, 'dual_text_pos_injection': True, 'bev_image_size': 448, 'vit_bev_freeze': True, 'vit_bev_use_lora': True, 'vit_bev_lora_rank': 64, 'vit_rgb_freeze': True, 'vit_rgb_use_lora': True, 'vit_rgb_lora_rank': 16, 'output_dir': '/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY', 'overwrite_output_dir': True, 'do_train': True, 'do_eval': False, 'do_predict': False, 'evaluation_strategy': 'no', 'prediction_loss_only': False, 'per_device_train_batch_size': 1, 'per_device_eval_batch_size': 8, 'per_gpu_train_batch_size': None, 'per_gpu_eval_batch_size': None, 'gradient_accumulation_steps': 1, 'eval_accumulation_steps': None, 'eval_delay': 0, 'learning_rate': 0.0001, 'weight_decay': 0.01, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_epsilon': 1e-08, 'max_grad_norm': 1.0, 'num_train_epochs': 1, 'max_steps': 9300, 'lr_scheduler_type': 'cosine', 'lr_scheduler_kwargs': {}, 'warmup_ratio': 0.03, 'warmup_steps': 0, 'log_level': 'passive', 'log_level_replica': 'warning', 'log_on_each_node': True, 'logging_dir': 'runs/Feb25_09-42-17_SH-IDC1-10-140-37-31', 'logging_strategy': 'steps', 'logging_first_step': False, 'logging_steps': 1, 'logging_nan_inf_filter': True, 'save_strategy': 'steps', 'save_steps': 0.5, 'save_total_limit': 2, 'save_safetensors': True, 'save_on_each_node': False, 'save_only_model': False, 'no_cuda': False, 'use_cpu': False, 'use_mps_device': False, 'seed': 42, 'data_seed': None, 'jit_mode_eval': False, 'use_ipex': False, 'bf16': True, 'fp16': False, 'fp16_opt_level': 'O1', 'half_precision_backend': 'auto', 'bf16_full_eval': False, 'fp16_full_eval': False, 'tf32': None, 'local_rank': 0, 'ddp_backend': None, 'tpu_num_cores': None, 'tpu_metrics_debug': False, 'debug': [], 'dataloader_drop_last': False, 'eval_steps': None, 'dataloader_num_workers': 2, 'past_index': -1, 'run_name': 'a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY_steps9300_gpus4_acc1', 'disable_tqdm': False, 'remove_unused_columns': False, 'label_names': None, 'load_best_model_at_end': False, 'metric_for_best_model': None, 'greater_is_better': None, 'ignore_data_skip': False, 'fsdp': [], 'fsdp_min_num_params': 0, 'fsdp_config': {'min_num_params': 0, 'xla': False, 'xla_fsdp_grad_ckpt': False}, 'fsdp_transformer_layer_cls_to_wrap': None, 'deepspeed': 'zero_stage2_config_acc1.json', 'label_smoothing_factor': 0.0, 'optim': 'adamw_torch', 'optim_args': None, 'adafactor': False, 'group_by_length': False, 'length_column_name': 'length', 'report_to': ['wandb'], 'ddp_find_unused_parameters': None, 'ddp_bucket_cap_mb': None, 'ddp_broadcast_buffers': None, 'dataloader_pin_memory': True, 'dataloader_persistent_workers': False, 'skip_memory_metrics': True, 'use_legacy_prediction_loop': False, 'push_to_hub': False, 'resume_from_checkpoint': None, 'hub_model_id': None, 'hub_strategy': 'every_save', 'hub_token': '<HUB_TOKEN>', 'hub_private_repo': False, 'hub_always_push': False, 'gradient_checkpointing': True, 'gradient_checkpointing_kwargs': None, 'include_inputs_for_metrics': False, 'fp16_backend': 'auto', 'push_to_hub_model_id': None, 'push_to_hub_organization': None, 'push_to_hub_token': '<PUSH_TO_HUB_TOKEN>', 'mp_parameters': '', 'auto_find_batch_size': False, 'full_determinism': False, 'torchdynamo': None, 'ray_scope': 'last', 'ddp_timeout': 1800, 'torch_compile': False, 'torch_compile_backend': None, 'torch_compile_mode': None, 'dispatch_batches': None, 'split_batches': False, 'include_tokens_per_second': False, 'include_num_input_tokens_seen': False, 'neftune_noise_alpha': None}

wandb/run-20260225_094307-hmhb8ltr/run-hmhb8ltr.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0beac92a32c1613762e91fc2beded5715934014b046b8dba88331bb08c80fa7
+size 1966080

wandb/run-20260225_111518-pg2w7c3p/files/config.yaml

@@ -0,0 +1,767 @@
+_attn_implementation_internal:
+    value: null
+_commit_hash:
+    value: null
+_name_or_path:
+    value: ../pretrained/InternVL3-2B
+_wandb:
+    value:
+        cli_version: 0.22.3
+        e:
+            hpt2lvw45stdjl0sezw62w5lspi4bemx:
+                args:
+                    - /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/resolved_train_config.json
+                codePath: InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py
+                codePathLocal: internvl_cleaned/train/internvl_chat_finetune.py
+                cpu_count: 64
+                cpu_count_logical: 128
+                cudaVersion: "12.2"
+                disk:
+                    /:
+                        total: "524945911808"
+                        used: "33262657536"
+                email: caiy0039@e.ntu.edu.sg
+                executable: /mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/bin/python3.10
+                git:
+                    commit: 003f6c01351c079c8c0476f53ac6212bc5573444
+                    remote: git@github.com:Yuxin916/CL_CoTNav.git
+                gpu: NVIDIA A100-SXM4-80GB
+                gpu_count: 8
+                gpu_nvidia:
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-0360b5f7-c6ac-dc17-a431-8ebfcce4c66d
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-56e8ab97-6913-5234-1a79-ff5ea4dfa2db
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-1dca27df-e145-5a54-2a2f-f5b58257f3ee
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-92c24b01-1351-1c21-e267-73234c73be2f
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-2af0eb8b-44c0-74eb-b3d3-d5107ccb6aa1
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-89cf5df2-4951-2e7b-3ee7-3d0fdafc5d07
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-f20f8083-ef3d-ef3f-7c2f-3244a690c7b6
+                    - architecture: Ampere
+                      cudaCores: 6912
+                      memoryTotal: "85899345920"
+                      name: NVIDIA A100-SXM4-80GB
+                      uuid: GPU-ab6cfaee-4d49-f6ee-3218-4f47bfeed414
+                host: SH-IDC1-10-140-37-90
+                memory:
+                    total: "1081627832320"
+                os: Linux-3.10.0-957.el7.x86_64-x86_64-with-glibc2.17
+                program: /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py
+                python: CPython 3.10.18
+                root: /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY
+                slurm:
+                    cluster_name: cluster_sproject3
+                    conf: /etc/slurm/slurm.conf
+                    cpus_on_node: "8"
+                    cpus_per_task: "8"
+                    distribution: cyclic
+                    gtids: "0"
+                    job_account: research
+                    job_cpus_per_node: "8"
+                    job_cpus_per_node_pack_group_0: "8"
+                    job_gid: "200000139"
+                    job_gpus: 0,1,2,3
+                    job_id: "7466527"
+                    job_name: vlm_ft
+                    job_nodelist: SH-IDC1-10-140-37-90
+                    job_num_nodes: "1"
+                    job_partition: interntmp
+                    job_qos: normal
+                    job_uid: "200000139"
+                    job_user: wangmaonan
+                    jobid: "7466527"
+                    launch_node_ipaddr: 10.140.37.90
+                    localid: "0"
+                    mem_per_node: "49152"
+                    nnodes: "1"
+                    node_aliases: (null)
+                    nodeid: "0"
+                    nodelist: SH-IDC1-10-140-37-90
+                    nprocs: "1"
+                    ntasks: "1"
+                    ntasks_per_node: "1"
+                    prio_process: "0"
+                    procid: "0"
+                    srun_comm_host: 10.140.37.90
+                    srun_comm_port: "43483"
+                    step_gpus: 0,1,2,3
+                    step_id: "0"
+                    step_launcher_port: "43483"
+                    step_nodelist: SH-IDC1-10-140-37-90
+                    step_num_nodes: "1"
+                    step_num_tasks: "1"
+                    step_tasks_per_node: "1"
+                    stepid: "0"
+                    submit_dir: /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav
+                    submit_host: SH-IDC1-10-140-37-90
+                    task_pid: "120843"
+                    tasks_per_node: "1"
+                    topology_addr: SH-IDC1-10-140-37-90
+                    topology_addr_pattern: node
+                    umask: "0002"
+                    working_cluster: cluster_sproject3:SH-IDC1-10-140-37-161:6817:9216:109
+                startedAt: "2026-02-25T03:15:18.014369Z"
+                writerId: hpt2lvw45stdjl0sezw62w5lspi4bemx
+        m:
+            - "1": train/global_step
+              "6":
+                - 3
+              "7": []
+            - "2": '*'
+              "5": 1
+              "6":
+                - 1
+              "7": []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 63
+                - 71
+                - 98
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 63
+                - 71
+                - 98
+                - 105
+            "3":
+                - 7
+                - 13
+                - 66
+            "4": 3.10.18
+            "5": 0.22.3
+            "6": 4.37.2
+            "9":
+                "1": transformers_trainer
+            "12": 0.22.3
+            "13": linux-x86_64
+adafactor:
+    value: false
+adam_beta1:
+    value: 0.9
+adam_beta2:
+    value: 0.999
+adam_epsilon:
+    value: 1e-08
+add_cross_attention:
+    value: false
+architectures:
+    value:
+        - InternVLChatModel
+auto_find_batch_size:
+    value: false
+auto_map:
+    value:
+        AutoConfig: configuration_internvl_chat.InternVLChatConfig
+        AutoModel: modeling_internvl_chat.InternVLChatModel
+        AutoModelForCausalLM: modeling_internvl_chat.InternVLChatModel
+bad_words_ids:
+    value: null
+begin_suppress_tokens:
+    value: null
+bev_image_size:
+    value: 448
+bf16:
+    value: true
+bf16_full_eval:
+    value: false
+bos_token_id:
+    value: null
+chunk_size_feed_forward:
+    value: 0
+cross_attention_hidden_size:
+    value: null
+data_seed:
+    value: null
+dataloader_drop_last:
+    value: false
+dataloader_num_workers:
+    value: 2
+dataloader_persistent_workers:
+    value: false
+dataloader_pin_memory:
+    value: true
+ddp_backend:
+    value: null
+ddp_broadcast_buffers:
+    value: null
+ddp_bucket_cap_mb:
+    value: null
+ddp_find_unused_parameters:
+    value: null
+ddp_timeout:
+    value: 1800
+debug:
+    value: []
+decoder_start_token_id:
+    value: null
+deepspeed:
+    value: zero_stage2_config_acc1.json
+disable_tqdm:
+    value: false
+dispatch_batches:
+    value: null
+diversity_penalty:
+    value: 0
+do_eval:
+    value: false
+do_predict:
+    value: false
+do_sample:
+    value: false
+do_train:
+    value: true
+downsample_ratio:
+    value: 0.5
+dual_text_pos_injection:
+    value: true
+dynamic_image_size:
+    value: false
+early_stopping:
+    value: false
+encoder_no_repeat_ngram_size:
+    value: 0
+eos_token_id:
+    value: null
+eval_accumulation_steps:
+    value: null
+eval_delay:
+    value: 0
+eval_steps:
+    value: null
+evaluation_strategy:
+    value: "no"
+exponential_decay_length_penalty:
+    value: null
+finetuning_task:
+    value: null
+force_image_size:
+    value: 448
+forced_bos_token_id:
+    value: null
+forced_eos_token_id:
+    value: null
+fp16:
+    value: false
+fp16_backend:
+    value: auto
+fp16_full_eval:
+    value: false
+fp16_opt_level:
+    value: O1
+fsdp:
+    value: []
+fsdp_config:
+    value:
+        min_num_params: 0
+        xla: false
+        xla_fsdp_grad_ckpt: false
+fsdp_min_num_params:
+    value: 0
+fsdp_transformer_layer_cls_to_wrap:
+    value: null
+full_determinism:
+    value: false
+gradient_accumulation_steps:
+    value: 1
+gradient_checkpointing:
+    value: true
+gradient_checkpointing_kwargs:
+    value: null
+greater_is_better:
+    value: null
+group_by_length:
+    value: false
+half_precision_backend:
+    value: auto
+hidden_size:
+    value: 1536
+hub_always_push:
+    value: false
+hub_model_id:
+    value: null
+hub_private_repo:
+    value: false
+hub_strategy:
+    value: every_save
+hub_token:
+    value: <HUB_TOKEN>
+id2label:
+    value:
+        "0": LABEL_0
+        "1": LABEL_1
+ignore_data_skip:
+    value: false
+image_fold:
+    value: null
+include_inputs_for_metrics:
+    value: false
+include_num_input_tokens_seen:
+    value: false
+include_tokens_per_second:
+    value: false
+is_decoder:
+    value: false
+is_encoder_decoder:
+    value: false
+jit_mode_eval:
+    value: false
+label_names:
+    value: null
+label_smoothing_factor:
+    value: 0
+label2id:
+    value:
+        LABEL_0: 0
+        LABEL_1: 1
+learning_rate:
+    value: 0.0001
+length_column_name:
+    value: length
+length_penalty:
+    value: 1
+llm_config:
+    value:
+        _attn_implementation_autoset: true
+        _name_or_path: ./pretrained/Qwen2.5-32B-Instruct
+        add_cross_attention: false
+        architectures:
+            - Qwen2ForCausalLM
+        attention_dropout: 0
+        attn_implementation: flash_attention_2
+        bad_words_ids: null
+        begin_suppress_tokens: null
+        bos_token_id: 151643
+        chunk_size_feed_forward: 0
+        cross_attention_hidden_size: null
+        decoder_start_token_id: null
+        diversity_penalty: 0
+        do_sample: false
+        early_stopping: false
+        encoder_no_repeat_ngram_size: 0
+        eos_token_id: 151643
+        exponential_decay_length_penalty: null
+        finetuning_task: null
+        forced_bos_token_id: null
+        forced_eos_token_id: null
+        hidden_act: silu
+        hidden_size: 1536
+        id2label:
+            "0": LABEL_0
+            "1": LABEL_1
+        initializer_range: 0.02
+        intermediate_size: 8960
+        is_decoder: false
+        is_encoder_decoder: false
+        label2id:
+            LABEL_0: 0
+            LABEL_1: 1
+        length_penalty: 1
+        max_length: 20
+        max_position_embeddings: 32768
+        max_window_layers: 70
+        min_length: 0
+        model_type: qwen2
+        moe_config: null
+        no_repeat_ngram_size: 0
+        num_attention_heads: 12
+        num_beam_groups: 1
+        num_beams: 1
+        num_hidden_layers: 28
+        num_key_value_heads: 2
+        num_return_sequences: 1
+        output_attentions: false
+        output_hidden_states: false
+        output_scores: false
+        pad_token_id: null
+        prefix: null
+        problem_type: null
+        remove_invalid_values: false
+        repetition_penalty: 1
+        return_dict: true
+        return_dict_in_generate: false
+        rms_norm_eps: 1e-06
+        rope_scaling:
+            factor: 2
+            rope_type: dynamic
+            type: dynamic
+        rope_theta: 1e+06
+        sep_token_id: null
+        sliding_window: null
+        suppress_tokens: null
+        task_specific_params: null
+        temperature: 1
+        tf_legacy_loss: false
+        tie_encoder_decoder: false
+        tie_word_embeddings: false
+        tokenizer_class: null
+        top_k: 50
+        top_p: 1
+        torch_dtype: bfloat16
+        torchscript: false
+        transformers_version: 4.37.2
+        typical_p: 1
+        use_bfloat16: true
+        use_cache: false
+        use_sliding_window: false
+        vocab_size: 151677
+load_best_model_at_end:
+    value: false
+local_rank:
+    value: 0
+log_level:
+    value: passive
+log_level_replica:
+    value: warning
+log_on_each_node:
+    value: true
+logging_dir:
+    value: runs/Feb25_11-14-26_SH-IDC1-10-140-37-90
+logging_first_step:
+    value: false
+logging_nan_inf_filter:
+    value: true
+logging_steps:
+    value: 1
+logging_strategy:
+    value: steps
+lr_scheduler_type:
+    value: cosine
+max_dynamic_patch:
+    value: 12
+max_grad_norm:
+    value: 1
+max_length:
+    value: 20
+max_steps:
+    value: 9300
+metric_for_best_model:
+    value: null
+min_dynamic_patch:
+    value: 1
+min_length:
+    value: 0
+model_type:
+    value: internvl_chat
+mp_parameters:
+    value: ""
+neftune_noise_alpha:
+    value: null
+no_cuda:
+    value: false
+no_repeat_ngram_size:
+    value: 0
+num_beam_groups:
+    value: 1
+num_beams:
+    value: 1
+num_image_token_bev:
+    value: 256
+num_image_token_ego:
+    value: 32
+num_return_sequences:
+    value: 1
+num_train_epochs:
+    value: 1
+optim:
+    value: adamw_torch
+optim_args:
+    value: null
+output_attentions:
+    value: false
+output_dir:
+    value: /mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY
+output_hidden_states:
+    value: false
+output_scores:
+    value: false
+overwrite_output_dir:
+    value: true
+pad_token_id:
+    value: null
+pad2square:
+    value: false
+past_index:
+    value: -1
+per_device_eval_batch_size:
+    value: 8
+per_device_train_batch_size:
+    value: 1
+per_gpu_eval_batch_size:
+    value: null
+per_gpu_train_batch_size:
+    value: null
+prediction_loss_only:
+    value: false
+prefix:
+    value: null
+problem_type:
+    value: null
+ps_version:
+    value: v2
+push_to_hub:
+    value: false
+push_to_hub_model_id:
+    value: null
+push_to_hub_organization:
+    value: null
+push_to_hub_token:
+    value: <PUSH_TO_HUB_TOKEN>
+ray_scope:
+    value: last
+remove_invalid_values:
+    value: false
+remove_unused_columns:
+    value: false
+repetition_penalty:
+    value: 1
+report_to:
+    value:
+        - wandb
+resume_from_checkpoint:
+    value: null
+return_dict:
+    value: true
+return_dict_in_generate:
+    value: false
+run_name:
+    value: a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY_steps9300_gpus4_acc1
+save_on_each_node:
+    value: false
+save_only_model:
+    value: false
+save_safetensors:
+    value: true
+save_steps:
+    value: 0.5
+save_strategy:
+    value: steps
+save_total_limit:
+    value: 1
+seed:
+    value: 42
+select_layer:
+    value: -1
+sep_token_id:
+    value: null
+skip_memory_metrics:
+    value: true
+split_batches:
+    value: false
+suppress_tokens:
+    value: null
+system_message:
+    value: You are an autonomous navigation agent operating in indoor environments. You receive spatial information through position embeddings injected into visual features and text tokens. Use the BEV map, position embeddings, and semantic information to make navigation decisions. When the target object is detected (<target> marker), navigate directly to it. Otherwise, explore frontiers strategically to find the goal object.
+task_specific_params:
+    value: null
+temperature:
+    value: 1
+template:
+    value: internvl2_5_nav
+tf_legacy_loss:
+    value: false
+tf32:
+    value: null
+tie_encoder_decoder:
+    value: false
+tie_word_embeddings:
+    value: false
+tokenizer_class:
+    value: null
+top_k:
+    value: 50
+top_p:
+    value: 1
+torch_compile:
+    value: false
+torch_compile_backend:
+    value: null
+torch_compile_mode:
+    value: null
+torch_dtype:
+    value: torch.bfloat16
+torchdynamo:
+    value: null
+torchscript:
+    value: false
+tpu_metrics_debug:
+    value: false
+tpu_num_cores:
+    value: null
+transformers_version:
+    value: null
+typical_p:
+    value: 1
+use_backbone_lora:
+    value: 0
+use_bfloat16:
+    value: false
+use_cpu:
+    value: false
+use_ipex:
+    value: false
+use_legacy_prediction_loop:
+    value: false
+use_llm_lora:
+    value: 64
+use_mps_device:
+    value: false
+use_pairwise_spatial_encoder:
+    value: false
+use_position_embeddings:
+    value: true
+use_thumbnail:
+    value: true
+vision_config:
+    value:
+        _attn_implementation_autoset: true
+        _name_or_path: OpenGVLab/InternViT-6B-448px-V1-5
+        add_cross_attention: false
+        architectures:
+            - InternVisionModel
+        attention_dropout: 0
+        auto_map:
+            AutoConfig: configuration_intern_vit.InternVisionConfig
+            AutoModel: modeling_intern_vit.InternVisionModel
+        bad_words_ids: null
+        begin_suppress_tokens: null
+        bos_token_id: null
+        capacity_factor: 1.2
+        chunk_size_feed_forward: 0
+        cross_attention_hidden_size: null
+        decoder_start_token_id: null
+        diversity_penalty: 0
+        do_sample: false
+        drop_path_rate: 0
+        dropout: 0
+        early_stopping: false
+        encoder_no_repeat_ngram_size: 0
+        eos_token_id: null
+        eval_capacity_factor: 1.4
+        exponential_decay_length_penalty: null
+        finetuning_task: null
+        forced_bos_token_id: null
+        forced_eos_token_id: null
+        hidden_act: gelu
+        hidden_size: 1024
+        id2label:
+            "0": LABEL_0
+            "1": LABEL_1
+        image_size: 448
+        initializer_factor: 0.1
+        initializer_range: 1e-10
+        intermediate_size: 4096
+        is_decoder: false
+        is_encoder_decoder: false
+        label2id:
+            LABEL_0: 0
+            LABEL_1: 1
+        laux_allreduce: all_nodes
+        layer_norm_eps: 1e-06
+        length_penalty: 1
+        max_length: 20
+        min_length: 0
+        model_type: intern_vit_6b
+        moe_coeff_ratio: 0.5
+        moe_intermediate_size: 768
+        moe_output_scale: 4
+        no_repeat_ngram_size: 0
+        noisy_gate_policy: RSample_before
+        norm_type: layer_norm
+        num_attention_heads: 16
+        num_beam_groups: 1
+        num_beams: 1
+        num_channels: 3
+        num_experts: 8
+        num_hidden_layers: 24
+        num_return_sequences: 1
+        num_routed_experts: 4
+        num_shared_experts: 4
+        output_attentions: false
+        output_hidden_states: false
+        output_scores: false
+        pad_token_id: null
+        patch_size: 14
+        prefix: null
+        problem_type: null
+        qk_normalization: false
+        qkv_bias: true
+        remove_invalid_values: false
+        repetition_penalty: 1
+        return_dict: true
+        return_dict_in_generate: false
+        sep_token_id: null
+        shared_expert_intermediate_size: 3072
+        suppress_tokens: null
+        task_specific_params: null
+        temperature: 1
+        tf_legacy_loss: false
+        tie_encoder_decoder: false
+        tie_word_embeddings: true
+        tokenizer_class: null
+        top_k: 50
+        top_p: 1
+        torch_dtype: bfloat16
+        torchscript: false
+        transformers_version: 4.37.2
+        typical_p: 1
+        use_bfloat16: true
+        use_flash_attn: true
+        use_moe: false
+        use_residual: true
+        use_rts: false
+        use_weighted_residual: false
+vit_bev_freeze:
+    value: true
+vit_bev_lora_rank:
+    value: 64
+vit_bev_use_lora:
+    value: true
+vit_rgb_freeze:
+    value: true
+vit_rgb_lora_rank:
+    value: 16
+vit_rgb_use_lora:
+    value: true
+warmup_ratio:
+    value: 0.03
+warmup_steps:
+    value: 0
+weight_decay:
+    value: 0.01

wandb/run-20260225_111518-pg2w7c3p/files/requirements.txt

@@ -0,0 +1,113 @@
+nvidia-cuda-runtime-cu12==12.1.105
+platformdirs==4.9.2
+wandb==0.22.3
+transformers==4.37.2
+einops==0.8.2
+accelerate==0.33.0
+aiosignal==1.4.0
+huggingface_hub==0.36.2
+pycparser==2.22
+multidict==6.7.1
+urllib3==2.6.3
+urllib3==2.5.0
+aiohappyeyeballs==2.6.1
+nvidia-cusolver-cu12==11.4.5.107
+yarl==1.22.0
+typing-inspection==0.4.2
+gitdb==4.0.12
+multiprocess==0.70.16
+async-timeout==5.0.1
+Jinja2==3.1.6
+setuptools==82.0.0
+torch==2.5.1+cu121
+hyperframe==6.1.0
+peft==0.10.0
+tzdata==2025.3
+dill==0.3.8
+orjson==3.11.7
+nvidia-cufft-cu12==11.0.2.54
+propcache==0.4.1
+PySocks==1.7.1
+nvidia-cublas-cu12==12.1.3.1
+nvidia-nccl-cu12==2.21.5
+docker-pycreds==0.4.0
+pydantic_core==2.41.5
+nvidia-ml-py==13.590.48
+ninja==1.13.0
+cffi==1.15.0
+nvidia-cuda-cupti-cu12==12.1.105
+triton==3.1.0
+annotated-types==0.7.0
+nvidia-nvtx-cu12==12.1.105
+timm==1.0.22
+sympy==1.13.1
+pydantic==2.12.5
+xxhash==3.6.0
+py-cpuinfo==9.0.0
+sentry-sdk==2.53.0
+networkx==3.4.2
+click==8.3.1
+regex==2026.1.15
+pillow==12.0.0
+zstandard==0.23.0
+pyarrow==20.0.0
+GitPython==3.1.46
+wheel==0.46.3
+fsspec==2025.3.0
+typing_extensions==4.15.0
+hf-xet==1.2.0
+torchvision==0.20.1+cu121
+python-dateutil==2.9.0.post0
+numpy==1.26.4
+smmap==5.0.2
+setproctitle==1.2.3
+safetensors==0.7.0
+requests==2.32.5
+hpack==4.1.0
+six==1.17.0
+modelscope==1.34.0
+pip==26.0.1
+h2==4.3.0
+PyYAML==6.0.3
+PyYAML==6.0
+torchaudio==2.5.1+cu121
+nvidia-nvjitlink-cu12==12.9.86
+psutil==7.2.2
+psutil==5.9.1
+pandas==2.3.2
+pytz==2025.2
+nvidia-cudnn-cu12==9.1.0.70
+packaging==25.0
+hjson==3.1.0
+deepspeed==0.14.4
+datasets==4.0.0
+nvidia-cuda-nvrtc-cu12==12.1.105
+tokenizers==0.15.2
+MarkupSafe==2.1.5
+charset-normalizer==3.4.4
+frozenlist==1.8.0
+Brotli==1.0.9
+certifi==2026.1.4
+mpmath==1.3.0
+protobuf==3.20.1
+tqdm==4.67.3
+nvidia-cusparse-cu12==12.1.0.106
+attrs==25.4.0
+appdirs==1.4.4
+flash_attn==2.7.4.post1
+aiohttp==3.13.3
+filelock==3.20.0
+idna==3.11
+nvidia-curand-cu12==10.3.2.106
+zipp==3.23.0
+backports.tarfile==1.2.0
+autocommand==2.2.2
+platformdirs==4.4.0
+jaraco.text==4.0.0
+more-itertools==10.8.0
+jaraco.context==6.1.0
+jaraco.functools==4.4.0
+wheel==0.46.3
+tomli==2.4.0
+packaging==26.0
+importlib_metadata==8.7.1

wandb/run-20260225_111518-pg2w7c3p/files/wandb-metadata.json

@@ -0,0 +1,144 @@
+{
+  "os":  "Linux-3.10.0-957.el7.x86_64-x86_64-with-glibc2.17",
+  "python":  "CPython 3.10.18",
+  "startedAt":  "2026-02-25T03:15:18.014369Z",
+  "args":  [
+    "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY/resolved_train_config.json"
+  ],
+  "program":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py",
+  "codePath":  "InternVL_cleaned/internvl_chat/internvl_cleaned/train/internvl_chat_finetune.py",
+  "codePathLocal":  "internvl_cleaned/train/internvl_chat_finetune.py",
+  "git":  {
+    "remote":  "git@github.com:Yuxin916/CL_CoTNav.git",
+    "commit":  "003f6c01351c079c8c0476f53ac6212bc5573444"
+  },
+  "email":  "caiy0039@e.ntu.edu.sg",
+  "root":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav/all_log/experiments/a100_dualvit_llm-64_mlp-train-patch-32768-acc1_BEVftFOV_FrontierRGB_PosB__FRONTIER_PIXEL_NUMBER_ONLY",
+  "host":  "SH-IDC1-10-140-37-90",
+  "executable":  "/mnt/petrelfs/wangmaonan/anaconda3/envs/cl_cotnav/bin/python3.10",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "NVIDIA A100-SXM4-80GB",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "524945911808",
+      "used":  "33262657536"
+    }
+  },
+  "memory":  {
+    "total":  "1081627832320"
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-0360b5f7-c6ac-dc17-a431-8ebfcce4c66d"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-56e8ab97-6913-5234-1a79-ff5ea4dfa2db"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-1dca27df-e145-5a54-2a2f-f5b58257f3ee"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-92c24b01-1351-1c21-e267-73234c73be2f"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-2af0eb8b-44c0-74eb-b3d3-d5107ccb6aa1"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-89cf5df2-4951-2e7b-3ee7-3d0fdafc5d07"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-f20f8083-ef3d-ef3f-7c2f-3244a690c7b6"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere",
+      "uuid":  "GPU-ab6cfaee-4d49-f6ee-3218-4f47bfeed414"
+    }
+  ],
+  "cudaVersion":  "12.2",
+  "slurm":  {
+    "cluster_name":  "cluster_sproject3",
+    "conf":  "/etc/slurm/slurm.conf",
+    "cpus_on_node":  "8",
+    "cpus_per_task":  "8",
+    "distribution":  "cyclic",
+    "gtids":  "0",
+    "job_account":  "research",
+    "job_cpus_per_node":  "8",
+    "job_cpus_per_node_pack_group_0":  "8",
+    "job_gid":  "200000139",
+    "job_gpus":  "0,1,2,3",
+    "job_id":  "7466527",
+    "job_name":  "vlm_ft",
+    "job_nodelist":  "SH-IDC1-10-140-37-90",
+    "job_num_nodes":  "1",
+    "job_partition":  "interntmp",
+    "job_qos":  "normal",
+    "job_uid":  "200000139",
+    "job_user":  "wangmaonan",
+    "jobid":  "7466527",
+    "launch_node_ipaddr":  "10.140.37.90",
+    "localid":  "0",
+    "mem_per_node":  "49152",
+    "nnodes":  "1",
+    "node_aliases":  "(null)",
+    "nodeid":  "0",
+    "nodelist":  "SH-IDC1-10-140-37-90",
+    "nprocs":  "1",
+    "ntasks":  "1",
+    "ntasks_per_node":  "1",
+    "prio_process":  "0",
+    "procid":  "0",
+    "srun_comm_host":  "10.140.37.90",
+    "srun_comm_port":  "43483",
+    "step_gpus":  "0,1,2,3",
+    "step_id":  "0",
+    "step_launcher_port":  "43483",
+    "step_nodelist":  "SH-IDC1-10-140-37-90",
+    "step_num_nodes":  "1",
+    "step_num_tasks":  "1",
+    "step_tasks_per_node":  "1",
+    "stepid":  "0",
+    "submit_dir":  "/mnt/petrelfs/wangmaonan/yuxin/CL_CoTNav",
+    "submit_host":  "SH-IDC1-10-140-37-90",
+    "task_pid":  "120843",
+    "tasks_per_node":  "1",
+    "topology_addr":  "SH-IDC1-10-140-37-90",
+    "topology_addr_pattern":  "node",
+    "umask":  "0002",
+    "working_cluster":  "cluster_sproject3:SH-IDC1-10-140-37-161:6817:9216:109"
+  },
+  "writerId":  "hpt2lvw45stdjl0sezw62w5lspi4bemx"
+}

wandb/run-20260225_111518-pg2w7c3p/files/wandb-summary.json

@@ -0,0 +1 @@
+{"train/train_runtime":117088.0105,"train/loss":0.1976,"train/global_step":9300,"_runtime":117087,"_timestamp":1.7721064053506212e+09,"train/learning_rate":0,"train/epoch":1,"train/train_steps_per_second":0.079,"train/total_flos":8.206174795797096e+22,"_wandb":{"runtime":117087},"train/train_loss":0.3191605252492171,"_step":9300,"train/train_samples_per_second":0.318}

wandb/run-20260225_111518-pg2w7c3p/logs/debug-core.log

@@ -0,0 +1,14 @@
+{"time":"2026-02-25T11:15:18.181861838+08:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpx01znwkb/port-121696.txt","pid":121696,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2026-02-25T11:15:18.184202356+08:00","level":"INFO","msg":"server: will exit if parent process dies","ppid":121696}
+{"time":"2026-02-25T11:15:18.184392132+08:00","level":"INFO","msg":"server: accepting connections","addr":{"Name":"/tmp/wandb-121696-125729-3028696784/socket","Net":"unix"}}
+{"time":"2026-02-25T11:15:18.255635297+08:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"1(@)"}
+{"time":"2026-02-25T11:15:18.265170365+08:00","level":"INFO","msg":"handleInformInit: received","streamId":"pg2w7c3p","id":"1(@)"}
+{"time":"2026-02-25T11:15:23.029413819+08:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"pg2w7c3p","id":"1(@)"}
+{"time":"2026-02-26T19:46:56.115178582+08:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"1(@)"}
+{"time":"2026-02-26T19:46:56.115812331+08:00","level":"INFO","msg":"connection: closing","id":"1(@)"}
+{"time":"2026-02-26T19:46:56.116231092+08:00","level":"INFO","msg":"connection: closed successfully","id":"1(@)"}
+{"time":"2026-02-26T19:46:56.115821395+08:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2026-02-26T19:46:56.117104362+08:00","level":"INFO","msg":"server: listener closed","addr":{"Name":"/tmp/wandb-121696-125729-3028696784/socket","Net":"unix"}}
+{"time":"2026-02-26T19:47:02.314061774+08:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"1(@)"}
+{"time":"2026-02-26T19:47:02.314530487+08:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"1(@)"}
+{"time":"2026-02-26T19:47:02.31490045+08:00","level":"INFO","msg":"server is closed"}

wandb/run-20260225_111518-pg2w7c3p/logs/debug-internal.log

@@ -0,0 +1,29 @@
+{"time":"2026-02-25T11:15:18.266363836+08:00","level":"INFO","msg":"stream: starting","core version":"0.22.3"}
+{"time":"2026-02-25T11:15:23.025454392+08:00","level":"INFO","msg":"stream: created new stream","id":"pg2w7c3p"}
+{"time":"2026-02-25T11:15:23.025989355+08:00","level":"INFO","msg":"handler: started","stream_id":"pg2w7c3p"}
+{"time":"2026-02-25T11:15:23.029061332+08:00","level":"INFO","msg":"stream: started","id":"pg2w7c3p"}
+{"time":"2026-02-25T11:15:23.029075662+08:00","level":"INFO","msg":"sender: started","stream_id":"pg2w7c3p"}
+{"time":"2026-02-25T11:15:23.02907461+08:00","level":"INFO","msg":"writer: started","stream_id":"pg2w7c3p"}
+{"time":"2026-02-25T15:48:54.519398391+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/tsaisplus-nanyang-technological-university-singapore/prompt_revision/pg2w7c3p/file_stream\": EOF"}
+{"time":"2026-02-25T21:07:59.713263866+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": unexpected EOF"}
+{"time":"2026-02-26T01:46:00.156791646+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":64931}
+{"time":"2026-02-26T01:46:01.26570749+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T01:46:04.155088853+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T02:27:44.815648016+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":68065}
+{"time":"2026-02-26T02:27:46.348757133+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T02:27:49.049539083+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T03:22:31.469120504+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T03:22:33.406774848+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":72171}
+{"time":"2026-02-26T03:22:34.088783654+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T08:26:46.733939909+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T08:26:53.80398635+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":95045}
+{"time":"2026-02-26T08:26:57.006489641+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T09:11:42.493210656+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/files/tsaisplus-nanyang-technological-university-singapore/prompt_revision/pg2w7c3p/file_stream\": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T12:21:30.540150918+08:00","level":"INFO","msg":"flowcontrol: backed up, offloading to disk","recordNumber":112659}
+{"time":"2026-02-26T12:21:31.920394554+08:00","level":"INFO","msg":"api: retrying error","error":"Post \"https://api.wandb.ai/graphql\": net/http: request canceled (Client.Timeout exceeded while awaiting headers)"}
+{"time":"2026-02-26T12:21:34.277988733+08:00","level":"INFO","msg":"flowcontrol: unblocked","totalOffloaded":13}
+{"time":"2026-02-26T19:46:56.115836343+08:00","level":"INFO","msg":"stream: closing","id":"pg2w7c3p"}
+{"time":"2026-02-26T19:47:01.871500678+08:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2026-02-26T19:47:02.310370252+08:00","level":"INFO","msg":"handler: closed","stream_id":"pg2w7c3p"}
+{"time":"2026-02-26T19:47:02.312032251+08:00","level":"INFO","msg":"sender: closed","stream_id":"pg2w7c3p"}
+{"time":"2026-02-26T19:47:02.312485027+08:00","level":"INFO","msg":"stream: closed","id":"pg2w7c3p"}